@leg3ndy/otto-bridge 0.5.4 → 0.5.6

package/dist/config.js

@@ -33,6 +33,26 @@ function sanitizePollIntervalMs(value, fallback = DEFAULT_CLAWD_CURSOR_POLL_INTE
     }
     return Math.max(250, Math.floor(parsed));
 }
+export function normalizeInstalledExtensions(values) {
+    if (!Array.isArray(values)) {
+        return [];
+    }
+    const seen = new Set();
+    const normalized = [];
+    for (const item of values) {
+        const slug = String(item || "")
+            .trim()
+            .toLowerCase()
+            .replace(/[^a-z0-9._-]+/g, "-")
+            .replace(/^-+|-+$/g, "");
+        if (!slug || seen.has(slug)) {
+            continue;
+        }
+        seen.add(slug);
+        normalized.push(slug);
+    }
+    return normalized;
+}
 function migrateLegacyExecutor(current) {
     if (platform() === "darwin"
         && current?.type === "clawd-cursor"
@@ -71,6 +91,7 @@ export async function loadBridgeConfig() {
             // The runtime must always report the currently installed package version.
             bridgeVersion: BRIDGE_VERSION,
             executor: resolveExecutorConfig(undefined, migrateLegacyExecutor(parsed.executor)),
+            installedExtensions: normalizeInstalledExtensions(parsed.installedExtensions),
         };
     }
     catch {
@@ -147,6 +168,7 @@ export function buildBridgeConfig(params) {
         approvalMode: params.approvalMode || "preview",
         capabilities: Array.isArray(params.capabilities) ? [...params.capabilities] : [],
         metadata: params.metadata || {},
+        installedExtensions: [],
         pairedAt: new Date().toISOString(),
         executor: resolveExecutorConfig(undefined, params.executor),
     };

package/dist/executors/native_macos.js

@@ -132,7 +132,132 @@ function extractMeaningfulDescriptionTokens(value) {
 function descriptionLikelyHasTextAnchor(description) {
     return extractQuotedPhrases(description).length > 0 || extractMeaningfulDescriptionTokens(description).length > 0;
 }
-function findOcrTextMatch(candidates, description) {
+function regionFromOcrItems(items, kind) {
+    if (!items.length) {
+        return null;
+    }
+    const sorted = [...items].sort((left, right) => {
+        if (left.y !== right.y)
+            return left.y - right.y;
+        return left.x - right.x;
+    });
+    const minX = Math.min(...sorted.map((item) => item.x));
+    const minY = Math.min(...sorted.map((item) => item.y));
+    const maxX = Math.max(...sorted.map((item) => item.x + item.width));
+    const maxY = Math.max(...sorted.map((item) => item.y + item.height));
+    const confidenceValues = sorted
+        .map((item) => Number(item.confidence))
+        .filter((value) => Number.isFinite(value));
+    const confidence = confidenceValues.length
+        ? confidenceValues.reduce((sum, value) => sum + value, 0) / confidenceValues.length
+        : undefined;
+    return {
+        text: sorted.map((item) => item.text).join(" ").replace(/\s+/g, " ").trim(),
+        x: minX,
+        y: minY,
+        width: Math.max(1, maxX - minX),
+        height: Math.max(1, maxY - minY),
+        confidence,
+        kind,
+    };
+}
+function buildStructuredOcrRegions(candidates) {
+    if (!candidates.length) {
+        return [];
+    }
+    const sorted = [...candidates].sort((left, right) => {
+        const leftCenterY = left.y + (left.height / 2);
+        const rightCenterY = right.y + (right.height / 2);
+        if (leftCenterY !== rightCenterY) {
+            return leftCenterY - rightCenterY;
+        }
+        return left.x - right.x;
+    });
+    const lines = [];
+    for (const candidate of sorted) {
+        const candidateCenterY = candidate.y + (candidate.height / 2);
+        const lastLine = lines[lines.length - 1];
+        if (!lastLine) {
+            lines.push([candidate]);
+            continue;
+        }
+        const referenceCenterY = lastLine.reduce((sum, item) => sum + item.y + (item.height / 2), 0) / lastLine.length;
+        const avgHeight = lastLine.reduce((sum, item) => sum + item.height, 0) / lastLine.length;
+        const maxDistance = Math.max(16, avgHeight * 0.75);
+        if (Math.abs(candidateCenterY - referenceCenterY) <= maxDistance) {
+            lastLine.push(candidate);
+        }
+        else {
+            lines.push([candidate]);
+        }
+    }
+    const lineRegions = lines
+        .map((line) => regionFromOcrItems(line.sort((left, right) => left.x - right.x), "line"))
+        .filter(Boolean);
+    const blocks = [];
+    for (const line of lineRegions) {
+        const lastBlock = blocks[blocks.length - 1];
+        if (!lastBlock) {
+            blocks.push([line]);
+            continue;
+        }
+        const previous = lastBlock[lastBlock.length - 1];
+        const verticalGap = line.y - (previous.y + previous.height);
+        const horizontalOverlap = Math.max(0, Math.min(previous.x + previous.width, line.x + line.width) - Math.max(previous.x, line.x));
+        const overlapRatio = horizontalOverlap / Math.max(previous.width, line.width, 1);
+        const leftAlignmentDelta = Math.abs(previous.x - line.x);
+        if (verticalGap <= Math.max(22, previous.height * 1.6) && (overlapRatio >= 0.18 || leftAlignmentDelta <= 120)) {
+            lastBlock.push(line);
+        }
+        else {
+            blocks.push([line]);
+        }
+    }
+    const blockRegions = blocks
+        .map((block) => {
+        const minX = Math.min(...block.map((line) => line.x));
+        const minY = Math.min(...block.map((line) => line.y));
+        const maxX = Math.max(...block.map((line) => line.x + line.width));
+        const maxY = Math.max(...block.map((line) => line.y + line.height));
+        const confidenceValues = block
+            .map((line) => Number(line.confidence))
+            .filter((value) => Number.isFinite(value));
+        const confidence = confidenceValues.length
+            ? confidenceValues.reduce((sum, value) => sum + value, 0) / confidenceValues.length
+            : undefined;
+        return {
+            text: block.map((line) => line.text).join(" ").replace(/\s+/g, " ").trim(),
+            x: minX,
+            y: minY,
+            width: Math.max(1, maxX - minX),
+            height: Math.max(1, maxY - minY),
+            confidence,
+            kind: "block",
+        };
+    })
+        .filter((region) => region.text);
+    const wordRegions = candidates.map((candidate) => ({
+        text: candidate.text,
+        x: candidate.x,
+        y: candidate.y,
+        width: candidate.width,
+        height: candidate.height,
+        confidence: candidate.confidence,
+        kind: "word",
+    }));
+    const unique = new Set();
+    const deduped = [];
+    for (const region of [...blockRegions, ...lineRegions, ...wordRegions]) {
+        const key = `${normalizeText(region.text)}|${Math.round(region.x)}|${Math.round(region.y)}|${region.kind}`;
+        if (!region.text || unique.has(key)) {
+            continue;
+        }
+        unique.add(key);
+        deduped.push(region);
+    }
+    return deduped;
+}
+function findOcrTextMatch(regions, description) {
     const phrases = extractQuotedPhrases(description);
     const tokens = extractMeaningfulDescriptionTokens(description);
     const normalizedDescription = normalizeText(description || "");
@@ -140,10 +265,11 @@ function findOcrTextMatch(candidates, description) {
     if (!phrases.length && !tokens.length) {
         return null;
     }
-    const scored = candidates
-        .map((candidate, index) => {
-        const normalizedText = normalizeText(candidate.text || "");
+    const scored = regions
+        .map((region, index) => {
+        const normalizedText = normalizeText(region.text || "");
         let score = 0;
+        let matchedTokens = 0;
         for (const phrase of phrases) {
             if (normalizedText.includes(phrase)) {
                 score += 120;
@@ -152,17 +278,27 @@ function findOcrTextMatch(candidates, description) {
         for (const token of tokens) {
             if (normalizedText.includes(token)) {
                 score += 18;
+                matchedTokens += 1;
             }
         }
+        if (tokens.length > 1 && matchedTokens === tokens.length) {
+            score += 36;
+        }
+        if (region.kind === "line") {
+            score += 8;
+        }
+        else if (region.kind === "block") {
+            score += 16;
+        }
         if (wantsFirst) {
-            score += Math.max(0, 24 - Math.round(candidate.y / 60));
+            score += Math.max(0, 24 - Math.round(region.y / 60));
             score += Math.max(0, 12 - index);
         }
-        if (candidate.confidence) {
-            score += Math.round(candidate.confidence * 20);
+        if (region.confidence) {
+            score += Math.round(region.confidence * 20);
         }
         return score > 0 ? {
-            candidate,
+            region,
             score,
         } : null;
     })
@@ -171,10 +307,10 @@ function findOcrTextMatch(candidates, description) {
         if (right.score !== left.score) {
             return right.score - left.score;
         }
-        if (left.candidate.y !== right.candidate.y) {
-            return left.candidate.y - right.candidate.y;
+        if (left.region.y !== right.region.y) {
+            return left.region.y - right.region.y;
         }
-        return left.candidate.x - right.candidate.x;
+        return left.region.x - right.region.x;
     });
     return scored[0] || null;
 }
@@ -288,6 +424,12 @@ function clipText(value, maxLength) {
     }
     return `${value.slice(0, maxLength)}...`;
 }
+function clipTextPreview(value, maxLength) {
+    if (value.length <= maxLength) {
+        return value;
+    }
+    return `${value.slice(0, maxLength)}\n\n[conteudo truncado: mostrando ${maxLength} de ${value.length} caracteres. Peca um trecho mais especifico se quiser continuar.]`;
+}
 const TEXTUTIL_READABLE_EXTENSIONS = new Set([
     ".doc",
     ".docx",
@@ -499,6 +641,15 @@ function parseStructuredActions(job) {
             actions.push({ type: "list_files", path: filePath, limit });
             continue;
         }
+        if (type === "count_files") {
+            const filePath = asString(action.path) || "~";
+            const extensions = Array.isArray(action.extensions)
+                ? action.extensions.map((item) => asString(item)?.toLowerCase().replace(/^\./, "")).filter(Boolean)
+                : undefined;
+            const recursive = typeof action.recursive === "boolean" ? action.recursive : undefined;
+            actions.push({ type: "count_files", path: filePath, extensions, recursive });
+            continue;
+        }
         if (type === "run_shell" || type === "shell" || type === "terminal") {
             const command = asString(action.command) || asString(action.cmd);
             const cwd = asString(action.cwd);
@@ -529,6 +680,19 @@ function parseStructuredActions(job) {
             }
             continue;
         }
+        if (type === "drag_visual_target" || type === "drag_target") {
+            const sourceDescription = asString(action.source_description) || asString(action.source) || asString(action.from);
+            const targetDescription = asString(action.target_description) || asString(action.target) || asString(action.to);
+            if (sourceDescription && targetDescription) {
+                actions.push({
+                    type: "drag_visual_target",
+                    source_description: sourceDescription,
+                    target_description: targetDescription,
+                    app: asString(action.app) || undefined,
+                });
+            }
+            continue;
+        }
     }
     return actions;
 }
@@ -746,6 +910,18 @@ export class NativeMacOSJobExecutor {
                     completionNotes.push(`Arquivos em ${action.path}:\n${listing}`);
                     continue;
                 }
+                if (action.type === "count_files") {
+                    await reporter.progress(progressPercent, `Contando arquivos em ${action.path}`);
+                    const counted = await this.countLocalFiles(action.path, action.extensions, action.recursive !== false);
+                    completionNotes.push(`Encontrei ${counted.total} arquivo${counted.total === 1 ? "" : "s"} ${counted.extensionsLabel} em ${counted.path}.`);
+                    resultPayload.file_count = {
+                        total: counted.total,
+                        path: counted.path,
+                        extensions: counted.extensions,
+                        recursive: counted.recursive,
+                    };
+                    continue;
+                }
                 if (action.type === "run_shell") {
                     await reporter.progress(progressPercent, `Rodando comando local: ${action.command}`);
                     const shellOutput = await this.runShellCommand(action.command, action.cwd);
@@ -833,15 +1009,15 @@ export class NativeMacOSJobExecutor {
                                 validated = true;
                             }
                             if (validated) {
-                                const candidate = ocrClick.candidate || null;
+                                const region = ocrClick.region || null;
                                 resultPayload.last_click = {
                                     strategy: ocrClick.strategy || "local_ocr",
                                     score: ocrClick.score || null,
-                                    matched_text: candidate?.text || null,
-                                    x: candidate ? candidate.x + (candidate.width / 2) : null,
-                                    y: candidate ? candidate.y + (candidate.height / 2) : null,
-                                    width: candidate?.width || null,
-                                    height: candidate?.height || null,
+                                    matched_text: region?.text || null,
+                                    x: region ? region.x + (region.width / 2) : null,
+                                    y: region ? region.y + (region.height / 2) : null,
+                                    width: region?.width || null,
+                                    height: region?.height || null,
                                 };
                                 completionNotes.push(`Localizei e cliquei em ${targetDescription} por OCR local.`);
                                 clickSucceeded = true;
@@ -917,6 +1093,35 @@ export class NativeMacOSJobExecutor {
                     }
                     continue;
                 }
+                if (action.type === "drag_visual_target") {
+                    const dragApp = await this.resolveLikelyBrowserApp(action.app);
+                    if (dragApp) {
+                        await reporter.progress(progressPercent, `Trazendo ${dragApp} para frente antes do arraste`);
+                        await this.focusApp(dragApp);
+                    }
+                    else if (action.app) {
+                        await reporter.progress(progressPercent, `Trazendo ${action.app} para frente antes do arraste`);
+                        await this.focusApp(action.app);
+                    }
+                    await reporter.progress(progressPercent, `Capturando a tela para localizar ${action.source_description} e ${action.target_description}`);
+                    const screenshotPath = await this.takeScreenshot();
+                    const sourcePoint = await this.resolveVisualTargetPoint(job.job_id, screenshotPath, action.source_description, artifacts, "drag_source");
+                    const targetPoint = await this.resolveVisualTargetPoint(job.job_id, screenshotPath, action.target_description, artifacts, "drag_target");
+                    if (!sourcePoint) {
+                        throw new Error(`Nao consegui localizar ${action.source_description} com confianca suficiente para arrastar.`);
+                    }
+                    if (!targetPoint) {
+                        throw new Error(`Nao consegui localizar ${action.target_description} com confianca suficiente para concluir o arraste.`);
+                    }
+                    await reporter.progress(progressPercent, `Arrastando ${action.source_description} para ${action.target_description}`);
+                    await this.dragPoint(sourcePoint.x, sourcePoint.y, targetPoint.x, targetPoint.y);
+                    resultPayload.last_drag = {
+                        source: sourcePoint,
+                        target: targetPoint,
+                    };
+                    completionNotes.push(`Arrastei ${action.source_description} para ${action.target_description}.`);
+                    continue;
+                }
                 await reporter.progress(progressPercent, `Abrindo ${action.url}${action.app ? ` em ${action.app}` : ""}`);
                 await this.openUrl(action.url, action.app);
                 await delay(1200);
@@ -1020,6 +1225,8 @@ end tell
             title: page.title,
             url: page.url,
             text: page.text,
+            playerTitle: page.playerTitle || "",
+            playerState: page.playerState || "",
         };
     }
     resolveExpectedBrowserHref(rawHref, baseUrl) {
@@ -1058,6 +1265,22 @@ end tell
                 return true;
             }
         }
+        const beforePlayerTitle = normalizeText(before?.playerTitle || "");
+        const afterPlayerTitle = normalizeText(after.playerTitle || "");
+        const beforePlayerState = normalizeText(before?.playerState || "");
+        const afterPlayerState = normalizeText(after.playerState || "");
+        const playerLooksActive = afterPlayerState.includes("pause") || afterPlayerState.includes("pausar");
+        if (afterUrl.includes("music.youtube.com")) {
+            if (beforePlayerState && afterPlayerState && beforePlayerState !== afterPlayerState && playerLooksActive) {
+                return true;
+            }
+            if (beforePlayerTitle && afterPlayerTitle && beforePlayerTitle !== afterPlayerTitle) {
+                return true;
+            }
+            if (!beforePlayerTitle && afterPlayerTitle && playerLooksActive) {
+                return true;
+            }
+        }
         const beforeTitle = normalizeText(before?.title || "");
         const afterTitle = normalizeText(after.title || "");
         if (beforeTitle && afterTitle && beforeTitle !== afterTitle) {
@@ -1259,6 +1482,7 @@ const normalize = (value) => String(value || "")
   .replace(/[\\u0300-\\u036f]/g, "")
   .toLowerCase();
 const normalizedDescription = normalize(rawDescription);
+const isYouTubeMusic = location.hostname.includes("music.youtube.com");
 const wantsFirst = /\\b(primeir[ao]?|first)\\b/.test(normalizedDescription);
 const wantsVideo = /\\b(video|videos|musica|faixa|youtube|resultado|watch)\\b/.test(normalizedDescription) || location.hostname.includes("youtube");
 const stopWords = new Set([
@@ -1275,7 +1499,19 @@ const tokens = Array.from(new Set(
     .filter((token) => token.length >= 3 && !stopWords.has(token))
 ));
 
-const candidateSelectors = location.hostname.includes("youtube")
+const candidateSelectors = isYouTubeMusic
+  ? [
+      "ytmusic-responsive-list-item-renderer a[href*='watch?v=']",
+      "ytmusic-responsive-list-item-renderer button[aria-label]",
+      "ytmusic-responsive-list-item-renderer tp-yt-paper-icon-button",
+      "ytmusic-responsive-list-item-renderer ytmusic-item-thumbnail-overlay-renderer button",
+      "ytmusic-shelf-renderer ytmusic-responsive-list-item-renderer a[href*='watch?v=']",
+      "a[href*='watch?v=']",
+      "button",
+      "[role='button']",
+      "[role='link']"
+    ]
+  : location.hostname.includes("youtube")
   ? [
       "ytd-video-renderer a#video-title",
       "ytd-video-renderer ytd-thumbnail a",
@@ -1339,8 +1575,11 @@ function scoreCandidate(element, rank) {
 
   if (wantsFirst) score += Math.max(0, 40 - rank);
   if (wantsVideo && normalizedHref.includes("/watch")) score += 30;
+  if (isYouTubeMusic && normalizedHref.includes("watch?v=")) score += 36;
+  if (isYouTubeMusic && element.closest("ytmusic-responsive-list-item-renderer, ytmusic-player-bar")) score += 24;
   if (location.hostname.includes("youtube") && element.closest("ytd-video-renderer, ytd-rich-item-renderer, ytd-rich-grid-media")) score += 20;
   if (element.id === "video-title") score += 12;
+  if (isYouTubeMusic && /\\b(play|pause|reproduzir|tocar)\\b/.test(normalizedText)) score += 12;
   if (!normalizedText && normalizedHref.includes("/watch")) score += 8;
 
   for (const phrase of quotedPhrases) {
@@ -1445,7 +1684,7 @@ tell application "Safari"
   activate
   if (count of windows) = 0 then error "Safari nao possui janelas abertas."
   delay 1
-  set pageJson to do JavaScript "(function(){const title=document.title||''; const url=location.href||''; const text=((document.body&&document.body.innerText)||'').trim().slice(0, 12000); return JSON.stringify({title:title,url:url,text:text});})();" in current tab of front window
+  set pageJson to do JavaScript "(function(){const title=document.title||''; const url=location.href||''; const text=((document.body&&document.body.innerText)||'').trim().slice(0, 12000); const playerButton=document.querySelector('ytmusic-player-bar #play-pause-button, ytmusic-player-bar tp-yt-paper-icon-button#play-pause-button, ytmusic-player-bar tp-yt-paper-icon-button.play-pause-button'); const playerTitle=(Array.from(document.querySelectorAll('ytmusic-player-bar .title, ytmusic-player-bar .content-info-wrapper .title, ytmusic-player-bar [slot=\"title\"]')).map((node)=>((node&&node.textContent)||'').trim()).find(Boolean))||''; const playerState=(playerButton&&((playerButton.getAttribute('title')||playerButton.getAttribute('aria-label')||playerButton.textContent)||'').trim())||''; return JSON.stringify({title:title,url:url,text:text,playerTitle:playerTitle,playerState:playerState});})();" in current tab of front window
 end tell
 return pageJson
 `;
@@ -1456,6 +1695,8 @@ return pageJson
                 title: asString(parsed.title) || "",
                 url: asString(parsed.url) || "",
                 text: asString(parsed.text) || "",
+                playerTitle: asString(parsed.playerTitle) || "",
+                playerState: asString(parsed.playerState) || "",
             };
         }
         catch (error) {
@@ -1478,6 +1719,8 @@ return pageTitle & linefeed & pageUrl
                 title: String(title || "").trim(),
                 url: String(url || "").trim(),
                 text: "",
+                playerTitle: "",
+                playerState: "",
             };
         }
     }
@@ -1523,6 +1766,54 @@ post(.leftMouseUp)
 `;
         await this.runCommand("swift", ["-e", script, String(Math.round(x)), String(Math.round(y))]);
     }
+    async dragPoint(fromX, fromY, toX, toY) {
+        const script = `
+import Cocoa
+import ApplicationServices
+
+let fromX = Double(CommandLine.arguments[1]) ?? 0
+let fromY = Double(CommandLine.arguments[2]) ?? 0
+let toX = Double(CommandLine.arguments[3]) ?? 0
+let toY = Double(CommandLine.arguments[4]) ?? 0
+
+let startPoint = CGPoint(x: fromX, y: fromY)
+let endPoint = CGPoint(x: toX, y: toY)
+let steps = max(8, Int(hypot(endPoint.x - startPoint.x, endPoint.y - startPoint.y) / 60.0))
+
+func post(_ type: CGEventType, at point: CGPoint) {
+  guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: point, mouseButton: .left) else {
+    fputs("failed to create mouse event\\n", stderr)
+    exit(1)
+  }
+  event.post(tap: .cghidEventTap)
+}
+
+post(.mouseMoved, at: startPoint)
+usleep(100000)
+post(.leftMouseDown, at: startPoint)
+usleep(90000)
+
+for step in 1...steps {
+  let progress = Double(step) / Double(steps)
+  let point = CGPoint(
+    x: startPoint.x + ((endPoint.x - startPoint.x) * progress),
+    y: startPoint.y + ((endPoint.y - startPoint.y) * progress)
+  )
+  post(.leftMouseDragged, at: point)
+  usleep(35000)
+}
+
+post(.leftMouseUp, at: endPoint)
+`;
+        await this.runCommand("swift", [
+            "-e",
+            script,
+            String(Math.round(fromX)),
+            String(Math.round(fromY)),
+            String(Math.round(toX)),
+            String(Math.round(toY)),
+        ]);
+    }
     async runLocalOcr(filePath) {
         const script = `
 import Foundation
@@ -1612,9 +1903,27 @@ if let output = String(data: data, encoding: .utf8) {
         }
     }
     async tryLocalOcrClick(screenshotPath, description) {
-        if (!descriptionLikelyHasTextAnchor(description)) {
+        const anchor = await this.resolveLocalOcrAnchor(screenshotPath, description);
+        if (!anchor.region) {
             return {
                 clicked: false,
+                reason: anchor.reason,
+                strategy: anchor.strategy,
+            };
+        }
+        const clickX = anchor.region.x + (anchor.region.width / 2);
+        const clickY = anchor.region.y + (anchor.region.height / 2);
+        await this.clickPoint(clickX, clickY);
+        return {
+            clicked: true,
+            score: anchor.score,
+            region: anchor.region,
+            strategy: anchor.strategy,
+        };
+    }
+    async resolveLocalOcrAnchor(screenshotPath, description) {
+        if (!descriptionLikelyHasTextAnchor(description)) {
+            return {
                 reason: "A descricao nao traz ancora textual forte para OCR local.",
                 strategy: "local_ocr_skipped",
             };
@@ -1622,27 +1931,69 @@ if let output = String(data: data, encoding: .utf8) {
         const candidates = await this.runLocalOcr(screenshotPath);
         if (!candidates.length) {
             return {
-                clicked: false,
                 reason: "OCR local nao encontrou texto utilizavel na tela.",
                 strategy: "local_ocr_empty",
             };
         }
-        const match = findOcrTextMatch(candidates, description);
+        const regions = buildStructuredOcrRegions(candidates);
+        const match = findOcrTextMatch(regions, description);
         if (!match || match.score < 24) {
             return {
-                clicked: false,
                 reason: "OCR local nao encontrou texto suficientemente compativel com a descricao.",
                 strategy: "local_ocr_no_match",
             };
         }
-        const clickX = match.candidate.x + (match.candidate.width / 2);
-        const clickY = match.candidate.y + (match.candidate.height / 2);
-        await this.clickPoint(clickX, clickY);
         return {
-            clicked: true,
+            region: match.region,
             score: match.score,
-            candidate: match.candidate,
-            strategy: "local_ocr",
+            strategy: match.region.kind === "block" ? "structured_local_ocr_block" : "local_ocr",
+        };
+    }
+    async resolveVisualTargetPoint(jobId, screenshotPath, description, artifacts, purpose) {
+        const ocrAnchor = await this.resolveLocalOcrAnchor(screenshotPath, description);
+        if (ocrAnchor.region) {
+            return {
+                x: ocrAnchor.region.x + (ocrAnchor.region.width / 2),
+                y: ocrAnchor.region.y + (ocrAnchor.region.height / 2),
+                strategy: ocrAnchor.strategy || "local_ocr",
+                matched_text: ocrAnchor.region.text,
+                score: ocrAnchor.score || null,
+            };
+        }
+        const uploadable = await this.buildUploadableImage(screenshotPath);
+        const artifact = await this.uploadArtifactForJob(jobId, uploadable.path, {
+            kind: "screenshot",
+            mimeTypeOverride: uploadable.mimeType,
+            fileNameOverride: uploadable.filename,
+            metadata: {
+                purpose,
+                visible_in_chat: false,
+                target: description,
+                width: uploadable.dimensions?.width || undefined,
+                height: uploadable.dimensions?.height || undefined,
+                original_width: uploadable.originalDimensions?.width || undefined,
+                original_height: uploadable.originalDimensions?.height || undefined,
+                resized_for_upload: uploadable.resized,
+            },
+        });
+        if (!artifact?.storage_path) {
+            return null;
+        }
+        artifacts.push(artifact);
+        const artifactMetadata = artifact.metadata || {};
+        const width = Number(artifactMetadata.width || 0);
+        const height = Number(artifactMetadata.height || 0);
+        const originalWidth = Number(artifactMetadata.original_width || width || 0);
+        const originalHeight = Number(artifactMetadata.original_height || height || 0);
+        const location = await this.locateVisualTarget(jobId, artifact.storage_path, description, width, height, artifact.mime_type);
+        if (!location?.found || typeof location.x !== "number" || typeof location.y !== "number") {
+            return null;
+        }
+        return {
+            x: width > 0 && originalWidth > 0 ? (location.x / width) * originalWidth : location.x,
+            y: height > 0 && originalHeight > 0 ? (location.y / height) * originalHeight : location.y,
+            strategy: "visual_locator",
+            score: typeof location.confidence === "number" ? location.confidence : null,
         };
     }
     async getImageDimensions(filePath) {
@@ -1715,7 +2066,7 @@ if let output = String(data: data, encoding: .utf8) {
             resized,
         };
     }
-    async readLocalFile(filePath, maxChars = 4000) {
+    async readLocalFile(filePath, maxChars = 1800) {
         const resolved = expandUserPath(filePath);
         const extension = path.extname(resolved).toLowerCase();
         if (TEXTUTIL_READABLE_EXTENSIONS.has(extension)) {
@@ -1726,7 +2077,7 @@ if let output = String(data: data, encoding: .utf8) {
                 resolved,
             ]);
             const content = sanitizeTextForJsonTransport(stdout);
-            return clipText(content || "(arquivo sem texto legivel)", maxChars);
+            return clipTextPreview(content || "(arquivo sem texto legivel)", maxChars);
         }
         const raw = await readFile(resolved);
         if (isLikelyBinaryBuffer(raw)) {
@@ -1735,7 +2086,7 @@ if let output = String(data: data, encoding: .utf8) {
             return clipText(`O arquivo ${filename} parece ser binario (${detectedType}) e nao pode ser lido como texto puro pelo Otto Bridge ainda.`, maxChars);
         }
         const content = sanitizeTextForJsonTransport(raw.toString("utf8"));
-        return clipText(content || "(arquivo vazio)", maxChars);
+        return clipTextPreview(content || "(arquivo vazio)", maxChars);
     }
     async listLocalFiles(directoryPath, limit = 40) {
         const resolved = expandUserPath(directoryPath);
@@ -1754,6 +2105,55 @@ if let output = String(data: data, encoding: .utf8) {
         }));
         return items.length > 0 ? items.join("\n") : "(pasta vazia)";
     }
+    async countLocalFiles(directoryPath, extensions, recursive = true) {
+        const resolved = expandUserPath(directoryPath);
+        const normalizedExtensions = Array.from(new Set((extensions || [])
+            .map((extension) => String(extension || "").trim().toLowerCase().replace(/^\./, ""))
+            .filter(Boolean)));
+        const queue = [resolved];
+        let total = 0;
+        while (queue.length > 0) {
+            const current = queue.shift();
+            if (!current)
+                continue;
+            let entries;
+            try {
+                entries = await readdir(current, { withFileTypes: true });
+            }
+            catch {
+                continue;
+            }
+            for (const entry of entries) {
+                const entryPath = path.join(current, entry.name);
+                if (entry.isDirectory()) {
+                    if (recursive) {
+                        queue.push(entryPath);
+                    }
+                    continue;
+                }
+                if (!entry.isFile()) {
+                    continue;
+                }
+                if (normalizedExtensions.length > 0) {
+                    const entryExtension = path.extname(entry.name).toLowerCase().replace(/^\./, "");
+                    if (!normalizedExtensions.includes(entryExtension)) {
+                        continue;
+                    }
+                }
+                total += 1;
+            }
+        }
+        const extensionsLabel = normalizedExtensions.length > 0
+            ? normalizedExtensions.map((extension) => `.${extension}`).join(", ")
+            : "do tipo solicitado";
+        return {
+            total,
+            path: directoryPath,
+            extensions: normalizedExtensions,
+            recursive,
+            extensionsLabel,
+        };
+    }
     async runShellCommand(command, cwd) {
         if (!isSafeShellCommand(command)) {
             throw new Error("Nenhum comando shell foi informado para execucao local.");
@@ -1805,6 +2205,9 @@ if let output = String(data: data, encoding: .utf8) {
         if (action.type === "list_files") {
             return `Arquivos listados em ${action.path}`;
         }
+        if (action.type === "count_files") {
+            return `Arquivos contados em ${action.path}`;
+        }
         if (action.type === "run_shell") {
             return `Comando ${action.command} executado no macOS`;
         }
@@ -1814,6 +2217,9 @@ if let output = String(data: data, encoding: .utf8) {
         if (action.type === "click_visual_target") {
             return `Clique guiado executado para ${action.description}`;
         }
+        if (action.type === "drag_visual_target") {
+            return `Arraste guiado executado de ${action.source_description} para ${action.target_description}`;
+        }
         const target = humanizeUrl(action.url);
         return `${target} foi aberto${action.app ? ` em ${action.app}` : ""}`;
     }

package/dist/main.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { spawn } from "node:child_process";
 import process from "node:process";
-import { clearBridgeConfig, getBridgeConfigPath, loadBridgeConfig, resolveApiBaseUrl, resolveExecutorConfig, } from "./config.js";
+import { clearBridgeConfig, getBridgeConfigPath, loadBridgeConfig, normalizeInstalledExtensions, resolveApiBaseUrl, resolveExecutorConfig, saveBridgeConfig, } from "./config.js";
 import { pairDevice } from "./pairing.js";
 import { BridgeRuntime } from "./runtime.js";
 import { BRIDGE_PACKAGE_NAME, BRIDGE_VERSION, DEFAULT_PAIR_TIMEOUT_SECONDS, DEFAULT_POLL_INTERVAL_MS, } from "./types.js";
@@ -56,6 +56,9 @@ function printUsage() {
   otto-bridge pair --api http://localhost:8000 --code ABC123 [--name "Meu PC"] [--executor native-macos|mock|clawd-cursor]
   otto-bridge run [--executor native-macos|mock|clawd-cursor] [--clawd-url http://127.0.0.1:3847]
   otto-bridge status
+  otto-bridge extensions --list
+  otto-bridge extensions --install github
+  otto-bridge extensions --uninstall github
   otto-bridge version
   otto-bridge update [--tag latest|next] [--dry-run]
   otto-bridge unpair
@@ -63,6 +66,8 @@ function printUsage() {
 Examples:
   otto-bridge pair --api https://api.leg3ndy.com.br --code ABC123
   otto-bridge run
+  otto-bridge extensions --install github
+  otto-bridge extensions --list
   otto-bridge version
   otto-bridge update
   otto-bridge update --dry-run
@@ -108,11 +113,15 @@ async function runPairCommand(args) {
     console.log(`[otto-bridge] config=${getBridgeConfigPath()}`);
     console.log("[otto-bridge] next step: run `otto-bridge run` to keep this device online");
 }
-async function runRuntimeCommand(args) {
+async function loadRequiredBridgeConfig() {
     const config = await loadBridgeConfig();
     if (!config) {
         throw new Error("No local pairing found. Run `otto-bridge pair --code <CODE>` first.");
     }
+    return config;
+}
+async function runRuntimeCommand(args) {
+    const config = await loadRequiredBridgeConfig();
     const runtimeConfig = {
         ...config,
         executor: resolveExecutorOverrides(args, config.executor),
@@ -136,10 +145,61 @@ async function runStatusCommand() {
         ws_url: config.wsUrl,
         approval_mode: config.approvalMode,
         capabilities: config.capabilities,
+        installed_extensions: config.installedExtensions,
         paired_at: config.pairedAt,
         executor: config.executor,
     }, null, 2));
 }
+async function runExtensionsCommand(args) {
+    const config = await loadRequiredBridgeConfig();
+    const installValue = option(args, "install");
+    const uninstallValue = option(args, "uninstall");
+    if (installValue && uninstallValue) {
+        throw new Error("Use apenas uma acao por vez: --install ou --uninstall.");
+    }
+    if (installValue) {
+        const nextExtensions = normalizeInstalledExtensions([
+            ...config.installedExtensions,
+            ...installValue.split(","),
+        ]);
+        const added = nextExtensions.filter((item) => !config.installedExtensions.includes(item));
+        if (!added.length) {
+            console.log("[otto-bridge] nenhuma extensao nova para instalar");
+            return;
+        }
+        await saveBridgeConfig({
+            ...config,
+            installedExtensions: nextExtensions,
+        });
+        console.log(`[otto-bridge] extensoes instaladas: ${added.join(", ")}`);
+        console.log("[otto-bridge] rode `otto-bridge run` novamente se quiser sincronizar agora com a web");
+        return;
+    }
+    if (uninstallValue) {
+        const removeSet = new Set(normalizeInstalledExtensions(uninstallValue.split(",")));
+        const nextExtensions = config.installedExtensions.filter((item) => !removeSet.has(item));
+        const removed = config.installedExtensions.filter((item) => removeSet.has(item));
+        if (!removed.length) {
+            console.log("[otto-bridge] nenhuma extensao correspondente estava instalada");
+            return;
+        }
+        await saveBridgeConfig({
+            ...config,
+            installedExtensions: nextExtensions,
+        });
+        console.log(`[otto-bridge] extensoes removidas: ${removed.join(", ")}`);
+        console.log("[otto-bridge] rode `otto-bridge run` novamente se quiser sincronizar agora com a web");
+        return;
+    }
+    if (!config.installedExtensions.length) {
+        console.log("[otto-bridge] nenhuma extensao instalada");
+        return;
+    }
+    console.log("[otto-bridge] extensoes instaladas:");
+    for (const extension of config.installedExtensions) {
+        console.log(`- ${extension}`);
+    }
+}
 async function runUnpairCommand() {
     await clearBridgeConfig();
     console.log("[otto-bridge] local pairing cleared");
@@ -173,6 +233,9 @@ async function main() {
         case "status":
             await runStatusCommand();
             return;
+        case "extensions":
+            await runExtensionsCommand(args);
+            return;
         case "version":
             printVersion();
             return;

package/dist/runtime.js

@@ -118,7 +118,10 @@ export class BridgeRuntime {
                     device_name: this.config.deviceName,
                     bridge_version: this.config.bridgeVersion,
                     capabilities: this.config.capabilities,
-                    metadata: this.config.metadata,
+                    metadata: {
+                        ...(this.config.metadata || {}),
+                        installed_extensions: this.config.installedExtensions,
+                    },
                 }));
                 heartbeatTimer = setInterval(() => {
                     if (socket.readyState === WebSocket.OPEN) {
@@ -163,7 +166,6 @@ export class BridgeRuntime {
         const type = String(message.type || "");
         switch (type) {
             case "device.hello":
-                this.maybeLogBridgeReleaseNotice(message);
                 console.log(`[otto-bridge] server hello device=${String(message.device_id || "")}`);
                 return;
             case "device.hello_ack":

package/dist/types.js

@@ -1,5 +1,5 @@
 export const BRIDGE_CONFIG_VERSION = 1;
-export const BRIDGE_VERSION = "0.5.4";
+export const BRIDGE_VERSION = "0.5.6";
 export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
 export const DEFAULT_API_BASE_URL = "http://localhost:8000";
 export const DEFAULT_POLL_INTERVAL_MS = 3000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leg3ndy/otto-bridge",
-  "version": "0.5.4",
+  "version": "0.5.6",
   "private": false,
   "type": "module",
   "description": "Local companion for Otto Bridge device pairing and WebSocket runtime.",