@leg3ndy/otto-bridge 0.5.5 → 0.5.8

package/dist/executors/native_macos.js

@@ -132,7 +132,132 @@ function extractMeaningfulDescriptionTokens(value) {
 function descriptionLikelyHasTextAnchor(description) {
     return extractQuotedPhrases(description).length > 0 || extractMeaningfulDescriptionTokens(description).length > 0;
 }
-function findOcrTextMatch(candidates, description) {
+function regionFromOcrItems(items, kind) {
+    if (!items.length) {
+        return null;
+    }
+    const sorted = [...items].sort((left, right) => {
+        if (left.y !== right.y)
+            return left.y - right.y;
+        return left.x - right.x;
+    });
+    const minX = Math.min(...sorted.map((item) => item.x));
+    const minY = Math.min(...sorted.map((item) => item.y));
+    const maxX = Math.max(...sorted.map((item) => item.x + item.width));
+    const maxY = Math.max(...sorted.map((item) => item.y + item.height));
+    const confidenceValues = sorted
+        .map((item) => Number(item.confidence))
+        .filter((value) => Number.isFinite(value));
+    const confidence = confidenceValues.length
+        ? confidenceValues.reduce((sum, value) => sum + value, 0) / confidenceValues.length
+        : undefined;
+    return {
+        text: sorted.map((item) => item.text).join(" ").replace(/\s+/g, " ").trim(),
+        x: minX,
+        y: minY,
+        width: Math.max(1, maxX - minX),
+        height: Math.max(1, maxY - minY),
+        confidence,
+        kind,
+    };
+}
+function buildStructuredOcrRegions(candidates) {
+    if (!candidates.length) {
+        return [];
+    }
+    const sorted = [...candidates].sort((left, right) => {
+        const leftCenterY = left.y + (left.height / 2);
+        const rightCenterY = right.y + (right.height / 2);
+        if (leftCenterY !== rightCenterY) {
+            return leftCenterY - rightCenterY;
+        }
+        return left.x - right.x;
+    });
+    const lines = [];
+    for (const candidate of sorted) {
+        const candidateCenterY = candidate.y + (candidate.height / 2);
+        const lastLine = lines[lines.length - 1];
+        if (!lastLine) {
+            lines.push([candidate]);
+            continue;
+        }
+        const referenceCenterY = lastLine.reduce((sum, item) => sum + item.y + (item.height / 2), 0) / lastLine.length;
+        const avgHeight = lastLine.reduce((sum, item) => sum + item.height, 0) / lastLine.length;
+        const maxDistance = Math.max(16, avgHeight * 0.75);
+        if (Math.abs(candidateCenterY - referenceCenterY) <= maxDistance) {
+            lastLine.push(candidate);
+        }
+        else {
+            lines.push([candidate]);
+        }
+    }
+    const lineRegions = lines
+        .map((line) => regionFromOcrItems(line.sort((left, right) => left.x - right.x), "line"))
+        .filter(Boolean);
+    const blocks = [];
+    for (const line of lineRegions) {
+        const lastBlock = blocks[blocks.length - 1];
+        if (!lastBlock) {
+            blocks.push([line]);
+            continue;
+        }
+        const previous = lastBlock[lastBlock.length - 1];
+        const verticalGap = line.y - (previous.y + previous.height);
+        const horizontalOverlap = Math.max(0, Math.min(previous.x + previous.width, line.x + line.width) - Math.max(previous.x, line.x));
+        const overlapRatio = horizontalOverlap / Math.max(previous.width, line.width, 1);
+        const leftAlignmentDelta = Math.abs(previous.x - line.x);
+        if (verticalGap <= Math.max(22, previous.height * 1.6) && (overlapRatio >= 0.18 || leftAlignmentDelta <= 120)) {
+            lastBlock.push(line);
+        }
+        else {
+            blocks.push([line]);
+        }
+    }
+    const blockRegions = blocks
+        .map((block) => {
+        const minX = Math.min(...block.map((line) => line.x));
+        const minY = Math.min(...block.map((line) => line.y));
+        const maxX = Math.max(...block.map((line) => line.x + line.width));
+        const maxY = Math.max(...block.map((line) => line.y + line.height));
+        const confidenceValues = block
+            .map((line) => Number(line.confidence))
+            .filter((value) => Number.isFinite(value));
+        const confidence = confidenceValues.length
+            ? confidenceValues.reduce((sum, value) => sum + value, 0) / confidenceValues.length
+            : undefined;
+        return {
+            text: block.map((line) => line.text).join(" ").replace(/\s+/g, " ").trim(),
+            x: minX,
+            y: minY,
+            width: Math.max(1, maxX - minX),
+            height: Math.max(1, maxY - minY),
+            confidence,
+            kind: "block",
+        };
+    })
+        .filter((region) => region.text);
+    const wordRegions = candidates.map((candidate) => ({
+        text: candidate.text,
+        x: candidate.x,
+        y: candidate.y,
+        width: candidate.width,
+        height: candidate.height,
+        confidence: candidate.confidence,
+        kind: "word",
+    }));
+    const unique = new Set();
+    const deduped = [];
+    for (const region of [...blockRegions, ...lineRegions, ...wordRegions]) {
+        const key = `${normalizeText(region.text)}|${Math.round(region.x)}|${Math.round(region.y)}|${region.kind}`;
+        if (!region.text || unique.has(key)) {
+            continue;
+        }
+        unique.add(key);
+        deduped.push(region);
+    }
+    return deduped;
+}
+function findOcrTextMatch(regions, description) {
     const phrases = extractQuotedPhrases(description);
     const tokens = extractMeaningfulDescriptionTokens(description);
     const normalizedDescription = normalizeText(description || "");
@@ -140,10 +265,11 @@ function findOcrTextMatch(candidates, description) {
     if (!phrases.length && !tokens.length) {
         return null;
     }
-    const scored = candidates
-        .map((candidate, index) => {
-        const normalizedText = normalizeText(candidate.text || "");
+    const scored = regions
+        .map((region, index) => {
+        const normalizedText = normalizeText(region.text || "");
         let score = 0;
+        let matchedTokens = 0;
         for (const phrase of phrases) {
             if (normalizedText.includes(phrase)) {
                 score += 120;
@@ -152,17 +278,27 @@ function findOcrTextMatch(candidates, description) {
         for (const token of tokens) {
             if (normalizedText.includes(token)) {
                 score += 18;
+                matchedTokens += 1;
             }
         }
+        if (tokens.length > 1 && matchedTokens === tokens.length) {
+            score += 36;
+        }
+        if (region.kind === "line") {
+            score += 8;
+        }
+        else if (region.kind === "block") {
+            score += 16;
+        }
         if (wantsFirst) {
-            score += Math.max(0, 24 - Math.round(candidate.y / 60));
+            score += Math.max(0, 24 - Math.round(region.y / 60));
             score += Math.max(0, 12 - index);
         }
-        if (candidate.confidence) {
-            score += Math.round(candidate.confidence * 20);
+        if (region.confidence) {
+            score += Math.round(region.confidence * 20);
         }
         return score > 0 ? {
-            candidate,
+            region,
             score,
         } : null;
     })
@@ -171,10 +307,10 @@ function findOcrTextMatch(candidates, description) {
         if (right.score !== left.score) {
             return right.score - left.score;
         }
-        if (left.candidate.y !== right.candidate.y) {
-            return left.candidate.y - right.candidate.y;
+        if (left.region.y !== right.region.y) {
+            return left.region.y - right.region.y;
         }
-        return left.candidate.x - right.candidate.x;
+        return left.region.x - right.region.x;
     });
     return scored[0] || null;
 }
@@ -505,6 +641,15 @@ function parseStructuredActions(job) {
             actions.push({ type: "list_files", path: filePath, limit });
             continue;
         }
+        if (type === "count_files") {
+            const filePath = asString(action.path) || "~";
+            const extensions = Array.isArray(action.extensions)
+                ? action.extensions.map((item) => asString(item)?.toLowerCase().replace(/^\./, "")).filter(Boolean)
+                : undefined;
+            const recursive = typeof action.recursive === "boolean" ? action.recursive : undefined;
+            actions.push({ type: "count_files", path: filePath, extensions, recursive });
+            continue;
+        }
         if (type === "run_shell" || type === "shell" || type === "terminal") {
             const command = asString(action.command) || asString(action.cmd);
             const cwd = asString(action.cwd);
@@ -520,6 +665,44 @@ function parseStructuredActions(job) {
             }
             continue;
         }
+        if (type === "scroll_view" || type === "scroll" || type === "scroll_page") {
+            const rawDirection = (asString(action.direction) || asString(action.dir) || "down").toLowerCase();
+            const direction = rawDirection === "up" || rawDirection === "cima" ? "up" : "down";
+            const rawAmount = (asString(action.amount) || "").toLowerCase();
+            const amount = rawAmount === "small" || rawAmount === "medium" || rawAmount === "large"
+                ? rawAmount
+                : undefined;
+            const rawSteps = Number(action.steps);
+            const steps = Number.isFinite(rawSteps) ? Math.max(1, Math.min(Math.round(rawSteps), 6)) : undefined;
+            actions.push({
+                type: "scroll_view",
+                direction,
+                amount,
+                steps,
+                app: asString(action.app) || undefined,
+            });
+            continue;
+        }
+        if (type === "whatsapp_send_message") {
+            const contact = asString(action.contact) || asString(action.recipient);
+            const text = asString(action.text) || asString(action.message);
+            if (contact && text) {
+                actions.push({ type: "whatsapp_send_message", contact, text });
+            }
+            continue;
+        }
+        if (type === "whatsapp_read_chat") {
+            const contact = asString(action.contact) || asString(action.recipient);
+            const rawLimit = Number(action.limit);
+            if (contact) {
+                actions.push({
+                    type: "whatsapp_read_chat",
+                    contact,
+                    limit: Number.isFinite(rawLimit) ? Math.max(1, Math.min(Math.round(rawLimit), 30)) : undefined,
+                });
+            }
+            continue;
+        }
         if (type === "click_visual_target" || type === "click_target") {
             const description = asString(action.description) || asString(action.target);
             if (description) {
@@ -535,6 +718,19 @@ function parseStructuredActions(job) {
             }
             continue;
         }
+        if (type === "drag_visual_target" || type === "drag_target") {
+            const sourceDescription = asString(action.source_description) || asString(action.source) || asString(action.from);
+            const targetDescription = asString(action.target_description) || asString(action.target) || asString(action.to);
+            if (sourceDescription && targetDescription) {
+                actions.push({
+                    type: "drag_visual_target",
+                    source_description: sourceDescription,
+                    target_description: targetDescription,
+                    app: asString(action.app) || undefined,
+                });
+            }
+            continue;
+        }
     }
     return actions;
 }
@@ -586,6 +782,23 @@ function deriveActionsFromText(job) {
         }
         return [{ type: "set_volume", level }];
     }
+    if (/\b(scroll|rola\w*|role\w*|desce\w*|sobe\w*)\b/i.test(task)) {
+        const direction = /\b(sobe\w*|suba\w*|para cima|pra cima|scroll up)\b/i.test(task) ? "up" : "down";
+        const amount = /\b(pouco|leve|small)\b/i.test(task)
+            ? "small"
+            : /\b(muito|bastante|fim|grande|large)\b/i.test(task)
+                ? "large"
+                : "medium";
+        const stepsMatch = task.match(/\b(\d{1,2})\s*(?:x|vezes?)\b/i);
+        const steps = stepsMatch?.[1] ? Math.max(1, Math.min(Number(stepsMatch[1]), 6)) : 1;
+        return [{
+                type: "scroll_view",
+                direction,
+                amount,
+                steps,
+                app: detectedApp || undefined,
+            }];
+    }
     if ((normalizedTask.includes("leia") || normalizedTask.includes("ler")) && detectedUrl) {
         return [
             { type: "open_url", url: detectedUrl, app: detectedApp || "Safari" },
@@ -752,6 +965,18 @@ export class NativeMacOSJobExecutor {
                     completionNotes.push(`Arquivos em ${action.path}:\n${listing}`);
                     continue;
                 }
+                if (action.type === "count_files") {
+                    await reporter.progress(progressPercent, `Contando arquivos em ${action.path}`);
+                    const counted = await this.countLocalFiles(action.path, action.extensions, action.recursive !== false);
+                    completionNotes.push(`Encontrei ${counted.total} arquivo${counted.total === 1 ? "" : "s"} ${counted.extensionsLabel} em ${counted.path}.`);
+                    resultPayload.file_count = {
+                        total: counted.total,
+                        path: counted.path,
+                        extensions: counted.extensions,
+                        recursive: counted.recursive,
+                    };
+                    continue;
+                }
                 if (action.type === "run_shell") {
                     await reporter.progress(progressPercent, `Rodando comando local: ${action.command}`);
                     const shellOutput = await this.runShellCommand(action.command, action.cwd);
@@ -764,6 +989,68 @@ export class NativeMacOSJobExecutor {
                     completionNotes.push(`Volume ajustado para ${action.level}% no macOS.`);
                     continue;
                 }
+                if (action.type === "scroll_view") {
+                    const scrollApp = action.app || this.lastActiveApp || await this.getFrontmostAppName();
+                    if (scrollApp) {
+                        await reporter.progress(progressPercent, `Trazendo ${scrollApp} para frente antes de rolar a tela`);
+                        await this.focusApp(scrollApp);
+                    }
+                    const directionLabel = action.direction === "up" ? "cima" : "baixo";
+                    await reporter.progress(progressPercent, `Rolando a tela para ${directionLabel}`);
+                    await this.scrollView(action.direction, action.amount, action.steps);
+                    resultPayload.last_scroll = {
+                        direction: action.direction,
+                        amount: action.amount || "medium",
+                        steps: action.steps || 1,
+                        app: scrollApp || null,
+                    };
+                    completionNotes.push(`Rolei a tela para ${directionLabel} no macOS.`);
+                    continue;
+                }
+                if (action.type === "whatsapp_send_message") {
+                    await reporter.progress(progressPercent, `Abrindo a conversa do WhatsApp com ${action.contact}`);
+                    await this.focusApp("Safari");
+                    await this.ensureWhatsAppWebReady();
+                    const selected = await this.selectWhatsAppConversation(action.contact);
+                    if (!selected) {
+                        throw new Error(`Nao consegui localizar a conversa do WhatsApp com ${action.contact}.`);
+                    }
+                    await reporter.progress(progressPercent, `Digitando a mensagem para ${action.contact} no WhatsApp`);
+                    await this.focusWhatsAppComposer();
+                    await this.typeText(action.text);
+                    await delay(250);
+                    await this.pressShortcut("return");
+                    await delay(900);
+                    const verification = await this.verifyWhatsAppLastMessage(action.text);
+                    if (!verification.ok) {
+                        throw new Error(verification.reason || `Nao consegui confirmar o envio da mensagem para ${action.contact} no WhatsApp.`);
+                    }
+                    resultPayload.whatsapp = {
+                        action: "send_message",
+                        contact: action.contact,
+                        text_preview: clipText(action.text, 180),
+                    };
+                    completionNotes.push(`Enviei a mensagem no WhatsApp para ${action.contact}.`);
+                    continue;
+                }
+                if (action.type === "whatsapp_read_chat") {
+                    await reporter.progress(progressPercent, `Abrindo a conversa do WhatsApp com ${action.contact}`);
+                    await this.focusApp("Safari");
+                    await this.ensureWhatsAppWebReady();
+                    const selected = await this.selectWhatsAppConversation(action.contact);
+                    if (!selected) {
+                        throw new Error(`Nao consegui localizar a conversa do WhatsApp com ${action.contact}.`);
+                    }
+                    await delay(500);
+                    const chat = await this.readWhatsAppVisibleConversation(action.contact, action.limit || 12);
+                    resultPayload.whatsapp = {
+                        action: "read_chat",
+                        contact: action.contact,
+                        messages: chat.messages,
+                    };
+                    completionNotes.push(`Mensagens visiveis no WhatsApp com ${action.contact}:\n${chat.summary}`);
+                    continue;
+                }
                 if (action.type === "click_visual_target") {
                     const browserApp = await this.resolveLikelyBrowserApp(action.app);
                     if (browserApp) {
@@ -839,15 +1126,15 @@ export class NativeMacOSJobExecutor {
                                 validated = true;
                             }
                             if (validated) {
-                                const candidate = ocrClick.candidate || null;
+                                const region = ocrClick.region || null;
                                 resultPayload.last_click = {
                                     strategy: ocrClick.strategy || "local_ocr",
                                     score: ocrClick.score || null,
-                                    matched_text: candidate?.text || null,
-                                    x: candidate ? candidate.x + (candidate.width / 2) : null,
-                                    y: candidate ? candidate.y + (candidate.height / 2) : null,
-                                    width: candidate?.width || null,
-                                    height: candidate?.height || null,
+                                    matched_text: region?.text || null,
+                                    x: region ? region.x + (region.width / 2) : null,
+                                    y: region ? region.y + (region.height / 2) : null,
+                                    width: region?.width || null,
+                                    height: region?.height || null,
                                 };
                                 completionNotes.push(`Localizei e cliquei em ${targetDescription} por OCR local.`);
                                 clickSucceeded = true;
@@ -923,6 +1210,35 @@ export class NativeMacOSJobExecutor {
                     }
                     continue;
                 }
+                if (action.type === "drag_visual_target") {
+                    const dragApp = await this.resolveLikelyBrowserApp(action.app);
+                    if (dragApp) {
+                        await reporter.progress(progressPercent, `Trazendo ${dragApp} para frente antes do arraste`);
+                        await this.focusApp(dragApp);
+                    }
+                    else if (action.app) {
+                        await reporter.progress(progressPercent, `Trazendo ${action.app} para frente antes do arraste`);
+                        await this.focusApp(action.app);
+                    }
+                    await reporter.progress(progressPercent, `Capturando a tela para localizar ${action.source_description} e ${action.target_description}`);
+                    const screenshotPath = await this.takeScreenshot();
+                    const sourcePoint = await this.resolveVisualTargetPoint(job.job_id, screenshotPath, action.source_description, artifacts, "drag_source");
+                    const targetPoint = await this.resolveVisualTargetPoint(job.job_id, screenshotPath, action.target_description, artifacts, "drag_target");
+                    if (!sourcePoint) {
+                        throw new Error(`Nao consegui localizar ${action.source_description} com confianca suficiente para arrastar.`);
+                    }
+                    if (!targetPoint) {
+                        throw new Error(`Nao consegui localizar ${action.target_description} com confianca suficiente para concluir o arraste.`);
+                    }
+                    await reporter.progress(progressPercent, `Arrastando ${action.source_description} para ${action.target_description}`);
+                    await this.dragPoint(sourcePoint.x, sourcePoint.y, targetPoint.x, targetPoint.y);
+                    resultPayload.last_drag = {
+                        source: sourcePoint,
+                        target: targetPoint,
+                    };
+                    completionNotes.push(`Arrastei ${action.source_description} para ${action.target_description}.`);
+                    continue;
+                }
                 await reporter.progress(progressPercent, `Abrindo ${action.url}${action.app ? ` em ${action.app}` : ""}`);
                 await this.openUrl(action.url, action.app);
                 await delay(1200);
@@ -1161,6 +1477,253 @@ end tell
             }
         }
     }
+    async scrollView(direction, amount = "medium", steps = 1) {
+        const clampedSteps = Math.max(1, Math.min(Math.round(steps || 1), 6));
+        const lineDelta = {
+            small: 4,
+            medium: 8,
+            large: 14,
+        }[amount];
+        const signedDelta = direction === "up" ? lineDelta : -lineDelta;
+        const iterations = {
+            small: 1,
+            medium: 2,
+            large: 3,
+        }[amount];
+        const swiftScript = `
+import ApplicationServices
+import Foundation
+
+let wheelDelta: Int32 = ${signedDelta}
+let stepCount = ${clampedSteps}
+let iterations = ${iterations}
+
+for _ in 0..<stepCount {
+  for _ in 0..<iterations {
+    if let event = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 1, wheel1: wheelDelta, wheel2: 0, wheel3: 0) {
+      event.post(tap: .cghidEventTap)
+    }
+    usleep(35000)
+  }
+  usleep(85000)
+}
+`;
+        try {
+            await this.runCommand("swift", ["-e", swiftScript]);
+        }
+        catch {
+            await this.scrollViewWithPageKeys(direction, clampedSteps);
+        }
+    }
+    async scrollViewWithPageKeys(direction, steps) {
+        const keyCode = direction === "up" ? 116 : 121;
+        const clampedSteps = Math.max(1, Math.min(Math.round(steps || 1), 6));
+        const script = `
+repeat ${clampedSteps} times
+  tell application "System Events" to key code ${keyCode}
+  delay 0.06
+end repeat
+`;
+        await this.runCommand("osascript", ["-e", script]);
+    }
+    async ensureWhatsAppWebReady() {
+        const page = await this.readFrontmostPage("Safari");
+        if (!normalizeComparableUrl(page.url || "").includes("web.whatsapp.com")) {
+            throw new Error("O Safari nao esta aberto no WhatsApp Web.");
+        }
+    }
+    async selectWhatsAppConversation(contact) {
+        const prepared = await this.runSafariJsonScript(`
+const query = String(__input?.contact || "");
+const normalize = (value) => String(value || "").normalize("NFD").replace(/[\\u0300-\\u036f]/g, "").toLowerCase().trim();
+const normalizedQuery = normalize(query);
+
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 4 || rect.height < 4) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+function focusAndReplaceContent(element, value) {
+  element.focus();
+  const range = document.createRange();
+  range.selectNodeContents(element);
+  const selection = window.getSelection();
+  selection?.removeAllRanges();
+  selection?.addRange(range);
+  document.execCommand("selectAll", false);
+  document.execCommand("delete", false);
+  document.execCommand("insertText", false, value);
+  element.dispatchEvent(new InputEvent("input", { bubbles: true, data: value, inputType: "insertText" }));
+}
+
+const candidates = Array.from(document.querySelectorAll('div[contenteditable="true"][role="textbox"], div[contenteditable="true"][data-tab], [data-testid="chat-list-search"] [contenteditable="true"]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node))
+  .map((node) => {
+    const element = node;
+    const rect = element.getBoundingClientRect();
+    const label = normalize(element.getAttribute("aria-label") || element.getAttribute("data-testid") || element.textContent || "");
+    let score = 0;
+    if (rect.left < window.innerWidth * 0.45) score += 30;
+    if (rect.top < 240) score += 30;
+    if (label.includes("search") || label.includes("pesquisar") || label.includes("procure") || label.includes("chat list")) score += 80;
+    if (element.closest('[data-testid="chat-list-search"], header')) score += 25;
+    return { element, score };
+  })
+  .sort((left, right) => right.score - left.score);
+
+if (!candidates.length) {
+  return { ok: false, reason: "Nao achei o campo de busca do WhatsApp Web." };
+}
+
+focusAndReplaceContent(candidates[0].element, query);
+return { ok: true };
+      `, { contact });
+        if (!prepared?.ok) {
+            return false;
+        }
+        await delay(900);
+        const result = await this.runSafariJsonScript(`
+const query = String(__input?.contact || "");
+const normalize = (value) => String(value || "").normalize("NFD").replace(/[\\u0300-\\u036f]/g, "").toLowerCase().trim();
+const normalizedQuery = normalize(query);
+
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 6 || rect.height < 6) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+const titleNodes = Array.from(document.querySelectorAll('span[title], div[title]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node))
+  .map((node) => {
+    const text = normalize(node.getAttribute("title") || node.textContent || "");
+    let score = 0;
+    if (text === normalizedQuery) score += 160;
+    if (text.includes(normalizedQuery)) score += 100;
+    if (normalizedQuery.includes(text) && text.length >= 3) score += 50;
+    const container = node.closest('[data-testid="cell-frame-container"], [role="listitem"], [role="gridcell"], div[tabindex]');
+    if (container instanceof HTMLElement && isVisible(container)) score += 20;
+    return { node, container, text, score };
+  })
+  .filter((item) => item.score > 0)
+  .sort((left, right) => right.score - left.score);
+
+if (!titleNodes.length) {
+  return { clicked: false, reason: "Nao achei uma conversa visivel com esse nome." };
+}
+
+const winner = titleNodes[0];
+const target = winner.container instanceof HTMLElement ? winner.container : winner.node;
+target.scrollIntoView({ block: "center", inline: "center", behavior: "auto" });
+target.dispatchEvent(new MouseEvent("mousedown", { bubbles: true, cancelable: true, view: window }));
+target.dispatchEvent(new MouseEvent("mouseup", { bubbles: true, cancelable: true, view: window }));
+target.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true, view: window }));
+if (typeof target.click === "function") {
+  target.click();
+}
+return { clicked: true };
+      `, { contact });
+        return Boolean(result?.clicked);
+    }
+    async focusWhatsAppComposer() {
+        const result = await this.runSafariJsonScript(`
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 6 || rect.height < 6) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+const candidates = Array.from(document.querySelectorAll('footer div[contenteditable="true"], [data-testid="conversation-compose-box-input"], main footer [contenteditable="true"]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node))
+  .sort((left, right) => right.getBoundingClientRect().top - left.getBoundingClientRect().top);
+
+if (!candidates.length) {
+  return { focused: false, reason: "Nao achei o campo de mensagem do WhatsApp Web." };
+}
+
+const composer = candidates[0];
+composer.focus();
+composer.click();
+return { focused: true };
+      `);
+        if (!result?.focused) {
+            throw new Error(result?.reason || "Nao consegui focar o campo de mensagem do WhatsApp Web.");
+        }
+    }
+    async readWhatsAppVisibleConversation(contact, limit) {
+        const result = await this.runSafariJsonScript(`
+const maxMessages = Number(__input?.limit || 12);
+
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 6 || rect.height < 6) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+const containers = Array.from(document.querySelectorAll('[data-testid="msg-container"], div[data-id]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node));
+
+const messages = containers.map((node) => {
+  const element = node;
+  const prePlain = element.querySelector('[data-pre-plain-text]')?.getAttribute('data-pre-plain-text') || "";
+  const authorMatch = prePlain.match(/\\]\\s*([^:]+):/);
+  const author = authorMatch?.[1]?.trim() || (element.getAttribute('data-testid')?.includes('out') ? 'Voce' : 'Contato');
+  const text = (element.innerText || "").trim().replace(/\\n{2,}/g, "\\n");
+  return { author, text };
+}).filter((item) => item.text);
+
+return { messages: messages.slice(-maxMessages) };
+      `, { contact, limit });
+        const messages = Array.isArray(result?.messages)
+            ? result.messages
+                .map((item) => ({
+                author: clipText(asString(item.author) || "Contato", 80),
+                text: clipText(asString(item.text) || "", 500),
+            }))
+                .filter((item) => item.text)
+            : [];
+        return {
+            messages,
+            summary: messages.length
+                ? messages.map((item) => `${item.author}: ${item.text}`).join("\n")
+                : `(sem mensagens visiveis na conversa com ${contact})`,
+        };
+    }
+    async verifyWhatsAppLastMessage(expectedText) {
+        const chat = await this.readWhatsAppVisibleConversation("Contato", 6);
+        if (!chat.messages.length) {
+            return {
+                ok: false,
+                reason: "Nao consegui ler as mensagens visiveis apos o envio no WhatsApp.",
+            };
+        }
+        const normalizedExpected = normalizeText(expectedText).slice(0, 60);
+        const matched = chat.messages.some((item) => normalizeText(item.text).includes(normalizedExpected));
+        if (!matched) {
+            return {
+                ok: false,
+                reason: "Nao consegui confirmar visualmente a mensagem enviada no WhatsApp.",
+            };
+        }
+        return { ok: true, reason: "" };
+    }
     async takeScreenshot(targetPath) {
         const artifactsDir = path.join(os.homedir(), ".otto-bridge", "artifacts");
         await mkdir(artifactsDir, { recursive: true });
@@ -1567,6 +2130,54 @@ post(.leftMouseUp)
 `;
         await this.runCommand("swift", ["-e", script, String(Math.round(x)), String(Math.round(y))]);
     }
+    async dragPoint(fromX, fromY, toX, toY) {
+        const script = `
+import Cocoa
+import ApplicationServices
+
+let fromX = Double(CommandLine.arguments[1]) ?? 0
+let fromY = Double(CommandLine.arguments[2]) ?? 0
+let toX = Double(CommandLine.arguments[3]) ?? 0
+let toY = Double(CommandLine.arguments[4]) ?? 0
+
+let startPoint = CGPoint(x: fromX, y: fromY)
+let endPoint = CGPoint(x: toX, y: toY)
+let steps = max(8, Int(hypot(endPoint.x - startPoint.x, endPoint.y - startPoint.y) / 60.0))
+
+func post(_ type: CGEventType, at point: CGPoint) {
+  guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: point, mouseButton: .left) else {
+    fputs("failed to create mouse event\\n", stderr)
+    exit(1)
+  }
+  event.post(tap: .cghidEventTap)
+}
+
+post(.mouseMoved, at: startPoint)
+usleep(100000)
+post(.leftMouseDown, at: startPoint)
+usleep(90000)
+
+for step in 1...steps {
+  let progress = Double(step) / Double(steps)
+  let point = CGPoint(
+    x: startPoint.x + ((endPoint.x - startPoint.x) * progress),
+    y: startPoint.y + ((endPoint.y - startPoint.y) * progress)
+  )
+  post(.leftMouseDragged, at: point)
+  usleep(35000)
+}
+
+post(.leftMouseUp, at: endPoint)
+`;
+        await this.runCommand("swift", [
+            "-e",
+            script,
+            String(Math.round(fromX)),
+            String(Math.round(fromY)),
+            String(Math.round(toX)),
+            String(Math.round(toY)),
+        ]);
+    }
     async runLocalOcr(filePath) {
         const script = `
 import Foundation
@@ -1656,9 +2267,27 @@ if let output = String(data: data, encoding: .utf8) {
         }
     }
     async tryLocalOcrClick(screenshotPath, description) {
-        if (!descriptionLikelyHasTextAnchor(description)) {
+        const anchor = await this.resolveLocalOcrAnchor(screenshotPath, description);
+        if (!anchor.region) {
             return {
                 clicked: false,
+                reason: anchor.reason,
+                strategy: anchor.strategy,
+            };
+        }
+        const clickX = anchor.region.x + (anchor.region.width / 2);
+        const clickY = anchor.region.y + (anchor.region.height / 2);
+        await this.clickPoint(clickX, clickY);
+        return {
+            clicked: true,
+            score: anchor.score,
+            region: anchor.region,
+            strategy: anchor.strategy,
+        };
+    }
+    async resolveLocalOcrAnchor(screenshotPath, description) {
+        if (!descriptionLikelyHasTextAnchor(description)) {
+            return {
                 reason: "A descricao nao traz ancora textual forte para OCR local.",
                 strategy: "local_ocr_skipped",
             };
@@ -1666,27 +2295,69 @@ if let output = String(data: data, encoding: .utf8) {
         const candidates = await this.runLocalOcr(screenshotPath);
         if (!candidates.length) {
             return {
-                clicked: false,
                 reason: "OCR local nao encontrou texto utilizavel na tela.",
                 strategy: "local_ocr_empty",
             };
         }
-        const match = findOcrTextMatch(candidates, description);
+        const regions = buildStructuredOcrRegions(candidates);
+        const match = findOcrTextMatch(regions, description);
         if (!match || match.score < 24) {
             return {
-                clicked: false,
                 reason: "OCR local nao encontrou texto suficientemente compativel com a descricao.",
                 strategy: "local_ocr_no_match",
             };
         }
-        const clickX = match.candidate.x + (match.candidate.width / 2);
-        const clickY = match.candidate.y + (match.candidate.height / 2);
-        await this.clickPoint(clickX, clickY);
         return {
-            clicked: true,
+            region: match.region,
             score: match.score,
-            candidate: match.candidate,
-            strategy: "local_ocr",
+            strategy: match.region.kind === "block" ? "structured_local_ocr_block" : "local_ocr",
+        };
+    }
+    async resolveVisualTargetPoint(jobId, screenshotPath, description, artifacts, purpose) {
+        const ocrAnchor = await this.resolveLocalOcrAnchor(screenshotPath, description);
+        if (ocrAnchor.region) {
+            return {
+                x: ocrAnchor.region.x + (ocrAnchor.region.width / 2),
+                y: ocrAnchor.region.y + (ocrAnchor.region.height / 2),
+                strategy: ocrAnchor.strategy || "local_ocr",
+                matched_text: ocrAnchor.region.text,
+                score: ocrAnchor.score || null,
+            };
+        }
+        const uploadable = await this.buildUploadableImage(screenshotPath);
+        const artifact = await this.uploadArtifactForJob(jobId, uploadable.path, {
+            kind: "screenshot",
+            mimeTypeOverride: uploadable.mimeType,
+            fileNameOverride: uploadable.filename,
+            metadata: {
+                purpose,
+                visible_in_chat: false,
+                target: description,
+                width: uploadable.dimensions?.width || undefined,
+                height: uploadable.dimensions?.height || undefined,
+                original_width: uploadable.originalDimensions?.width || undefined,
+                original_height: uploadable.originalDimensions?.height || undefined,
+                resized_for_upload: uploadable.resized,
+            },
+        });
+        if (!artifact?.storage_path) {
+            return null;
+        }
+        artifacts.push(artifact);
+        const artifactMetadata = artifact.metadata || {};
+        const width = Number(artifactMetadata.width || 0);
+        const height = Number(artifactMetadata.height || 0);
+        const originalWidth = Number(artifactMetadata.original_width || width || 0);
+        const originalHeight = Number(artifactMetadata.original_height || height || 0);
+        const location = await this.locateVisualTarget(jobId, artifact.storage_path, description, width, height, artifact.mime_type);
+        if (!location?.found || typeof location.x !== "number" || typeof location.y !== "number") {
+            return null;
+        }
+        return {
+            x: width > 0 && originalWidth > 0 ? (location.x / width) * originalWidth : location.x,
+            y: height > 0 && originalHeight > 0 ? (location.y / height) * originalHeight : location.y,
+            strategy: "visual_locator",
+            score: typeof location.confidence === "number" ? location.confidence : null,
         };
     }
     async getImageDimensions(filePath) {
@@ -1798,6 +2469,55 @@ if let output = String(data: data, encoding: .utf8) {
         }));
         return items.length > 0 ? items.join("\n") : "(pasta vazia)";
     }
+    async countLocalFiles(directoryPath, extensions, recursive = true) {
+        const resolved = expandUserPath(directoryPath);
+        const normalizedExtensions = Array.from(new Set((extensions || [])
+            .map((extension) => String(extension || "").trim().toLowerCase().replace(/^\./, ""))
+            .filter(Boolean)));
+        const queue = [resolved];
+        let total = 0;
+        while (queue.length > 0) {
+            const current = queue.shift();
+            if (!current)
+                continue;
+            let entries;
+            try {
+                entries = await readdir(current, { withFileTypes: true });
+            }
+            catch {
+                continue;
+            }
+            for (const entry of entries) {
+                const entryPath = path.join(current, entry.name);
+                if (entry.isDirectory()) {
+                    if (recursive) {
+                        queue.push(entryPath);
+                    }
+                    continue;
+                }
+                if (!entry.isFile()) {
+                    continue;
+                }
+                if (normalizedExtensions.length > 0) {
+                    const entryExtension = path.extname(entry.name).toLowerCase().replace(/^\./, "");
+                    if (!normalizedExtensions.includes(entryExtension)) {
+                        continue;
+                    }
+                }
+                total += 1;
+            }
+        }
+        const extensionsLabel = normalizedExtensions.length > 0
+            ? normalizedExtensions.map((extension) => `.${extension}`).join(", ")
+            : "do tipo solicitado";
+        return {
+            total,
+            path: directoryPath,
+            extensions: normalizedExtensions,
+            recursive,
+            extensionsLabel,
+        };
+    }
     async runShellCommand(command, cwd) {
         if (!isSafeShellCommand(command)) {
             throw new Error("Nenhum comando shell foi informado para execucao local.");
@@ -1849,15 +2569,30 @@ if let output = String(data: data, encoding: .utf8) {
         if (action.type === "list_files") {
             return `Arquivos listados em ${action.path}`;
         }
+        if (action.type === "count_files") {
+            return `Arquivos contados em ${action.path}`;
+        }
         if (action.type === "run_shell") {
             return `Comando ${action.command} executado no macOS`;
         }
         if (action.type === "set_volume") {
             return `Volume ajustado para ${action.level}% no macOS`;
         }
+        if (action.type === "scroll_view") {
+            return `Tela rolada para ${action.direction === "up" ? "cima" : "baixo"} no macOS`;
+        }
+        if (action.type === "whatsapp_send_message") {
+            return `Mensagem enviada no WhatsApp para ${action.contact}`;
+        }
+        if (action.type === "whatsapp_read_chat") {
+            return `Conversa do WhatsApp lida com ${action.contact}`;
+        }
         if (action.type === "click_visual_target") {
             return `Clique guiado executado para ${action.description}`;
         }
+        if (action.type === "drag_visual_target") {
+            return `Arraste guiado executado de ${action.source_description} para ${action.target_description}`;
+        }
         const target = humanizeUrl(action.url);
         return `${target} foi aberto${action.app ? ` em ${action.app}` : ""}`;
     }

package/dist/types.js

@@ -1,5 +1,5 @@
 export const BRIDGE_CONFIG_VERSION = 1;
-export const BRIDGE_VERSION = "0.5.5";
+export const BRIDGE_VERSION = "0.5.8";
 export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
 export const DEFAULT_API_BASE_URL = "http://localhost:8000";
 export const DEFAULT_POLL_INTERVAL_MS = 3000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leg3ndy/otto-bridge",
-  "version": "0.5.5",
+  "version": "0.5.8",
   "private": false,
   "type": "module",
   "description": "Local companion for Otto Bridge device pairing and WebSocket runtime.",