@leg3ndy/otto-bridge 0.5.5 → 0.5.6

package/dist/executors/native_macos.js

@@ -132,7 +132,132 @@ function extractMeaningfulDescriptionTokens(value) {
 function descriptionLikelyHasTextAnchor(description) {
     return extractQuotedPhrases(description).length > 0 || extractMeaningfulDescriptionTokens(description).length > 0;
 }
-function findOcrTextMatch(candidates, description) {
+function regionFromOcrItems(items, kind) {
+    if (!items.length) {
+        return null;
+    }
+    const sorted = [...items].sort((left, right) => {
+        if (left.y !== right.y)
+            return left.y - right.y;
+        return left.x - right.x;
+    });
+    const minX = Math.min(...sorted.map((item) => item.x));
+    const minY = Math.min(...sorted.map((item) => item.y));
+    const maxX = Math.max(...sorted.map((item) => item.x + item.width));
+    const maxY = Math.max(...sorted.map((item) => item.y + item.height));
+    const confidenceValues = sorted
+        .map((item) => Number(item.confidence))
+        .filter((value) => Number.isFinite(value));
+    const confidence = confidenceValues.length
+        ? confidenceValues.reduce((sum, value) => sum + value, 0) / confidenceValues.length
+        : undefined;
+    return {
+        text: sorted.map((item) => item.text).join(" ").replace(/\s+/g, " ").trim(),
+        x: minX,
+        y: minY,
+        width: Math.max(1, maxX - minX),
+        height: Math.max(1, maxY - minY),
+        confidence,
+        kind,
+    };
+}
+function buildStructuredOcrRegions(candidates) {
+    if (!candidates.length) {
+        return [];
+    }
+    const sorted = [...candidates].sort((left, right) => {
+        const leftCenterY = left.y + (left.height / 2);
+        const rightCenterY = right.y + (right.height / 2);
+        if (leftCenterY !== rightCenterY) {
+            return leftCenterY - rightCenterY;
+        }
+        return left.x - right.x;
+    });
+    const lines = [];
+    for (const candidate of sorted) {
+        const candidateCenterY = candidate.y + (candidate.height / 2);
+        const lastLine = lines[lines.length - 1];
+        if (!lastLine) {
+            lines.push([candidate]);
+            continue;
+        }
+        const referenceCenterY = lastLine.reduce((sum, item) => sum + item.y + (item.height / 2), 0) / lastLine.length;
+        const avgHeight = lastLine.reduce((sum, item) => sum + item.height, 0) / lastLine.length;
+        const maxDistance = Math.max(16, avgHeight * 0.75);
+        if (Math.abs(candidateCenterY - referenceCenterY) <= maxDistance) {
+            lastLine.push(candidate);
+        }
+        else {
+            lines.push([candidate]);
+        }
+    }
+    const lineRegions = lines
+        .map((line) => regionFromOcrItems(line.sort((left, right) => left.x - right.x), "line"))
+        .filter(Boolean);
+    const blocks = [];
+    for (const line of lineRegions) {
+        const lastBlock = blocks[blocks.length - 1];
+        if (!lastBlock) {
+            blocks.push([line]);
+            continue;
+        }
+        const previous = lastBlock[lastBlock.length - 1];
+        const verticalGap = line.y - (previous.y + previous.height);
+        const horizontalOverlap = Math.max(0, Math.min(previous.x + previous.width, line.x + line.width) - Math.max(previous.x, line.x));
+        const overlapRatio = horizontalOverlap / Math.max(previous.width, line.width, 1);
+        const leftAlignmentDelta = Math.abs(previous.x - line.x);
+        if (verticalGap <= Math.max(22, previous.height * 1.6) && (overlapRatio >= 0.18 || leftAlignmentDelta <= 120)) {
+            lastBlock.push(line);
+        }
+        else {
+            blocks.push([line]);
+        }
+    }
+    const blockRegions = blocks
+        .map((block) => {
+        const minX = Math.min(...block.map((line) => line.x));
+        const minY = Math.min(...block.map((line) => line.y));
+        const maxX = Math.max(...block.map((line) => line.x + line.width));
+        const maxY = Math.max(...block.map((line) => line.y + line.height));
+        const confidenceValues = block
+            .map((line) => Number(line.confidence))
+            .filter((value) => Number.isFinite(value));
+        const confidence = confidenceValues.length
+            ? confidenceValues.reduce((sum, value) => sum + value, 0) / confidenceValues.length
+            : undefined;
+        return {
+            text: block.map((line) => line.text).join(" ").replace(/\s+/g, " ").trim(),
+            x: minX,
+            y: minY,
+            width: Math.max(1, maxX - minX),
+            height: Math.max(1, maxY - minY),
+            confidence,
+            kind: "block",
+        };
+    })
+        .filter((region) => region.text);
+    const wordRegions = candidates.map((candidate) => ({
+        text: candidate.text,
+        x: candidate.x,
+        y: candidate.y,
+        width: candidate.width,
+        height: candidate.height,
+        confidence: candidate.confidence,
+        kind: "word",
+    }));
+    const unique = new Set();
+    const deduped = [];
+    for (const region of [...blockRegions, ...lineRegions, ...wordRegions]) {
+        const key = `${normalizeText(region.text)}|${Math.round(region.x)}|${Math.round(region.y)}|${region.kind}`;
+        if (!region.text || unique.has(key)) {
+            continue;
+        }
+        unique.add(key);
+        deduped.push(region);
+    }
+    return deduped;
+}
+function findOcrTextMatch(regions, description) {
     const phrases = extractQuotedPhrases(description);
     const tokens = extractMeaningfulDescriptionTokens(description);
     const normalizedDescription = normalizeText(description || "");
@@ -140,10 +265,11 @@ function findOcrTextMatch(candidates, description) {
     if (!phrases.length && !tokens.length) {
         return null;
     }
-    const scored = candidates
-        .map((candidate, index) => {
-        const normalizedText = normalizeText(candidate.text || "");
+    const scored = regions
+        .map((region, index) => {
+        const normalizedText = normalizeText(region.text || "");
         let score = 0;
+        let matchedTokens = 0;
         for (const phrase of phrases) {
             if (normalizedText.includes(phrase)) {
                 score += 120;
@@ -152,17 +278,27 @@ function findOcrTextMatch(candidates, description) {
         for (const token of tokens) {
             if (normalizedText.includes(token)) {
                 score += 18;
+                matchedTokens += 1;
             }
         }
+        if (tokens.length > 1 && matchedTokens === tokens.length) {
+            score += 36;
+        }
+        if (region.kind === "line") {
+            score += 8;
+        }
+        else if (region.kind === "block") {
+            score += 16;
+        }
         if (wantsFirst) {
-            score += Math.max(0, 24 - Math.round(candidate.y / 60));
+            score += Math.max(0, 24 - Math.round(region.y / 60));
             score += Math.max(0, 12 - index);
         }
-        if (candidate.confidence) {
-            score += Math.round(candidate.confidence * 20);
+        if (region.confidence) {
+            score += Math.round(region.confidence * 20);
         }
         return score > 0 ? {
-            candidate,
+            region,
             score,
         } : null;
     })
@@ -171,10 +307,10 @@ function findOcrTextMatch(candidates, description) {
         if (right.score !== left.score) {
             return right.score - left.score;
         }
-        if (left.candidate.y !== right.candidate.y) {
-            return left.candidate.y - right.candidate.y;
+        if (left.region.y !== right.region.y) {
+            return left.region.y - right.region.y;
         }
-        return left.candidate.x - right.candidate.x;
+        return left.region.x - right.region.x;
     });
     return scored[0] || null;
 }
@@ -505,6 +641,15 @@ function parseStructuredActions(job) {
             actions.push({ type: "list_files", path: filePath, limit });
             continue;
         }
+        if (type === "count_files") {
+            const filePath = asString(action.path) || "~";
+            const extensions = Array.isArray(action.extensions)
+                ? action.extensions.map((item) => asString(item)?.toLowerCase().replace(/^\./, "")).filter(Boolean)
+                : undefined;
+            const recursive = typeof action.recursive === "boolean" ? action.recursive : undefined;
+            actions.push({ type: "count_files", path: filePath, extensions, recursive });
+            continue;
+        }
         if (type === "run_shell" || type === "shell" || type === "terminal") {
             const command = asString(action.command) || asString(action.cmd);
             const cwd = asString(action.cwd);
@@ -535,6 +680,19 @@ function parseStructuredActions(job) {
             }
             continue;
         }
+        if (type === "drag_visual_target" || type === "drag_target") {
+            const sourceDescription = asString(action.source_description) || asString(action.source) || asString(action.from);
+            const targetDescription = asString(action.target_description) || asString(action.target) || asString(action.to);
+            if (sourceDescription && targetDescription) {
+                actions.push({
+                    type: "drag_visual_target",
+                    source_description: sourceDescription,
+                    target_description: targetDescription,
+                    app: asString(action.app) || undefined,
+                });
+            }
+            continue;
+        }
     }
     return actions;
 }
@@ -752,6 +910,18 @@ export class NativeMacOSJobExecutor {
                     completionNotes.push(`Arquivos em ${action.path}:\n${listing}`);
                     continue;
                 }
+                if (action.type === "count_files") {
+                    await reporter.progress(progressPercent, `Contando arquivos em ${action.path}`);
+                    const counted = await this.countLocalFiles(action.path, action.extensions, action.recursive !== false);
+                    completionNotes.push(`Encontrei ${counted.total} arquivo${counted.total === 1 ? "" : "s"} ${counted.extensionsLabel} em ${counted.path}.`);
+                    resultPayload.file_count = {
+                        total: counted.total,
+                        path: counted.path,
+                        extensions: counted.extensions,
+                        recursive: counted.recursive,
+                    };
+                    continue;
+                }
                 if (action.type === "run_shell") {
                     await reporter.progress(progressPercent, `Rodando comando local: ${action.command}`);
                     const shellOutput = await this.runShellCommand(action.command, action.cwd);
@@ -839,15 +1009,15 @@ export class NativeMacOSJobExecutor {
                                 validated = true;
                             }
                             if (validated) {
-                                const candidate = ocrClick.candidate || null;
+                                const region = ocrClick.region || null;
                                 resultPayload.last_click = {
                                     strategy: ocrClick.strategy || "local_ocr",
                                     score: ocrClick.score || null,
-                                    matched_text: candidate?.text || null,
-                                    x: candidate ? candidate.x + (candidate.width / 2) : null,
-                                    y: candidate ? candidate.y + (candidate.height / 2) : null,
-                                    width: candidate?.width || null,
-                                    height: candidate?.height || null,
+                                    matched_text: region?.text || null,
+                                    x: region ? region.x + (region.width / 2) : null,
+                                    y: region ? region.y + (region.height / 2) : null,
+                                    width: region?.width || null,
+                                    height: region?.height || null,
                                 };
                                 completionNotes.push(`Localizei e cliquei em ${targetDescription} por OCR local.`);
                                 clickSucceeded = true;
@@ -923,6 +1093,35 @@ export class NativeMacOSJobExecutor {
                     }
                     continue;
                 }
+                if (action.type === "drag_visual_target") {
+                    const dragApp = await this.resolveLikelyBrowserApp(action.app);
+                    if (dragApp) {
+                        await reporter.progress(progressPercent, `Trazendo ${dragApp} para frente antes do arraste`);
+                        await this.focusApp(dragApp);
+                    }
+                    else if (action.app) {
+                        await reporter.progress(progressPercent, `Trazendo ${action.app} para frente antes do arraste`);
+                        await this.focusApp(action.app);
+                    }
+                    await reporter.progress(progressPercent, `Capturando a tela para localizar ${action.source_description} e ${action.target_description}`);
+                    const screenshotPath = await this.takeScreenshot();
+                    const sourcePoint = await this.resolveVisualTargetPoint(job.job_id, screenshotPath, action.source_description, artifacts, "drag_source");
+                    const targetPoint = await this.resolveVisualTargetPoint(job.job_id, screenshotPath, action.target_description, artifacts, "drag_target");
+                    if (!sourcePoint) {
+                        throw new Error(`Nao consegui localizar ${action.source_description} com confianca suficiente para arrastar.`);
+                    }
+                    if (!targetPoint) {
+                        throw new Error(`Nao consegui localizar ${action.target_description} com confianca suficiente para concluir o arraste.`);
+                    }
+                    await reporter.progress(progressPercent, `Arrastando ${action.source_description} para ${action.target_description}`);
+                    await this.dragPoint(sourcePoint.x, sourcePoint.y, targetPoint.x, targetPoint.y);
+                    resultPayload.last_drag = {
+                        source: sourcePoint,
+                        target: targetPoint,
+                    };
+                    completionNotes.push(`Arrastei ${action.source_description} para ${action.target_description}.`);
+                    continue;
+                }
                 await reporter.progress(progressPercent, `Abrindo ${action.url}${action.app ? ` em ${action.app}` : ""}`);
                 await this.openUrl(action.url, action.app);
                 await delay(1200);
@@ -1567,6 +1766,54 @@ post(.leftMouseUp)
 `;
         await this.runCommand("swift", ["-e", script, String(Math.round(x)), String(Math.round(y))]);
     }
+    async dragPoint(fromX, fromY, toX, toY) {
+        const script = `
+import Cocoa
+import ApplicationServices
+
+let fromX = Double(CommandLine.arguments[1]) ?? 0
+let fromY = Double(CommandLine.arguments[2]) ?? 0
+let toX = Double(CommandLine.arguments[3]) ?? 0
+let toY = Double(CommandLine.arguments[4]) ?? 0
+
+let startPoint = CGPoint(x: fromX, y: fromY)
+let endPoint = CGPoint(x: toX, y: toY)
+let steps = max(8, Int(hypot(endPoint.x - startPoint.x, endPoint.y - startPoint.y) / 60.0))
+
+func post(_ type: CGEventType, at point: CGPoint) {
+  guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: point, mouseButton: .left) else {
+    fputs("failed to create mouse event\\n", stderr)
+    exit(1)
+  }
+  event.post(tap: .cghidEventTap)
+}
+
+post(.mouseMoved, at: startPoint)
+usleep(100000)
+post(.leftMouseDown, at: startPoint)
+usleep(90000)
+
+for step in 1...steps {
+  let progress = Double(step) / Double(steps)
+  let point = CGPoint(
+    x: startPoint.x + ((endPoint.x - startPoint.x) * progress),
+    y: startPoint.y + ((endPoint.y - startPoint.y) * progress)
+  )
+  post(.leftMouseDragged, at: point)
+  usleep(35000)
+}
+
+post(.leftMouseUp, at: endPoint)
+`;
+        await this.runCommand("swift", [
+            "-e",
+            script,
+            String(Math.round(fromX)),
+            String(Math.round(fromY)),
+            String(Math.round(toX)),
+            String(Math.round(toY)),
+        ]);
+    }
     async runLocalOcr(filePath) {
         const script = `
 import Foundation
@@ -1656,9 +1903,27 @@ if let output = String(data: data, encoding: .utf8) {
         }
     }
     async tryLocalOcrClick(screenshotPath, description) {
-        if (!descriptionLikelyHasTextAnchor(description)) {
+        const anchor = await this.resolveLocalOcrAnchor(screenshotPath, description);
+        if (!anchor.region) {
             return {
                 clicked: false,
+                reason: anchor.reason,
+                strategy: anchor.strategy,
+            };
+        }
+        const clickX = anchor.region.x + (anchor.region.width / 2);
+        const clickY = anchor.region.y + (anchor.region.height / 2);
+        await this.clickPoint(clickX, clickY);
+        return {
+            clicked: true,
+            score: anchor.score,
+            region: anchor.region,
+            strategy: anchor.strategy,
+        };
+    }
+    async resolveLocalOcrAnchor(screenshotPath, description) {
+        if (!descriptionLikelyHasTextAnchor(description)) {
+            return {
                 reason: "A descricao nao traz ancora textual forte para OCR local.",
                 strategy: "local_ocr_skipped",
             };
@@ -1666,27 +1931,69 @@ if let output = String(data: data, encoding: .utf8) {
         const candidates = await this.runLocalOcr(screenshotPath);
         if (!candidates.length) {
             return {
-                clicked: false,
                 reason: "OCR local nao encontrou texto utilizavel na tela.",
                 strategy: "local_ocr_empty",
             };
         }
-        const match = findOcrTextMatch(candidates, description);
+        const regions = buildStructuredOcrRegions(candidates);
+        const match = findOcrTextMatch(regions, description);
         if (!match || match.score < 24) {
             return {
-                clicked: false,
                 reason: "OCR local nao encontrou texto suficientemente compativel com a descricao.",
                 strategy: "local_ocr_no_match",
             };
         }
-        const clickX = match.candidate.x + (match.candidate.width / 2);
-        const clickY = match.candidate.y + (match.candidate.height / 2);
-        await this.clickPoint(clickX, clickY);
         return {
-            clicked: true,
+            region: match.region,
             score: match.score,
-            candidate: match.candidate,
-            strategy: "local_ocr",
+            strategy: match.region.kind === "block" ? "structured_local_ocr_block" : "local_ocr",
+        };
+    }
+    async resolveVisualTargetPoint(jobId, screenshotPath, description, artifacts, purpose) {
+        const ocrAnchor = await this.resolveLocalOcrAnchor(screenshotPath, description);
+        if (ocrAnchor.region) {
+            return {
+                x: ocrAnchor.region.x + (ocrAnchor.region.width / 2),
+                y: ocrAnchor.region.y + (ocrAnchor.region.height / 2),
+                strategy: ocrAnchor.strategy || "local_ocr",
+                matched_text: ocrAnchor.region.text,
+                score: ocrAnchor.score || null,
+            };
+        }
+        const uploadable = await this.buildUploadableImage(screenshotPath);
+        const artifact = await this.uploadArtifactForJob(jobId, uploadable.path, {
+            kind: "screenshot",
+            mimeTypeOverride: uploadable.mimeType,
+            fileNameOverride: uploadable.filename,
+            metadata: {
+                purpose,
+                visible_in_chat: false,
+                target: description,
+                width: uploadable.dimensions?.width || undefined,
+                height: uploadable.dimensions?.height || undefined,
+                original_width: uploadable.originalDimensions?.width || undefined,
+                original_height: uploadable.originalDimensions?.height || undefined,
+                resized_for_upload: uploadable.resized,
+            },
+        });
+        if (!artifact?.storage_path) {
+            return null;
+        }
+        artifacts.push(artifact);
+        const artifactMetadata = artifact.metadata || {};
+        const width = Number(artifactMetadata.width || 0);
+        const height = Number(artifactMetadata.height || 0);
+        const originalWidth = Number(artifactMetadata.original_width || width || 0);
+        const originalHeight = Number(artifactMetadata.original_height || height || 0);
+        const location = await this.locateVisualTarget(jobId, artifact.storage_path, description, width, height, artifact.mime_type);
+        if (!location?.found || typeof location.x !== "number" || typeof location.y !== "number") {
+            return null;
+        }
+        return {
+            x: width > 0 && originalWidth > 0 ? (location.x / width) * originalWidth : location.x,
+            y: height > 0 && originalHeight > 0 ? (location.y / height) * originalHeight : location.y,
+            strategy: "visual_locator",
+            score: typeof location.confidence === "number" ? location.confidence : null,
         };
     }
     async getImageDimensions(filePath) {
@@ -1798,6 +2105,55 @@ if let output = String(data: data, encoding: .utf8) {
         }));
         return items.length > 0 ? items.join("\n") : "(pasta vazia)";
     }
+    async countLocalFiles(directoryPath, extensions, recursive = true) {
+        const resolved = expandUserPath(directoryPath);
+        const normalizedExtensions = Array.from(new Set((extensions || [])
+            .map((extension) => String(extension || "").trim().toLowerCase().replace(/^\./, ""))
+            .filter(Boolean)));
+        const queue = [resolved];
+        let total = 0;
+        while (queue.length > 0) {
+            const current = queue.shift();
+            if (!current)
+                continue;
+            let entries;
+            try {
+                entries = await readdir(current, { withFileTypes: true });
+            }
+            catch {
+                continue;
+            }
+            for (const entry of entries) {
+                const entryPath = path.join(current, entry.name);
+                if (entry.isDirectory()) {
+                    if (recursive) {
+                        queue.push(entryPath);
+                    }
+                    continue;
+                }
+                if (!entry.isFile()) {
+                    continue;
+                }
+                if (normalizedExtensions.length > 0) {
+                    const entryExtension = path.extname(entry.name).toLowerCase().replace(/^\./, "");
+                    if (!normalizedExtensions.includes(entryExtension)) {
+                        continue;
+                    }
+                }
+                total += 1;
+            }
+        }
+        const extensionsLabel = normalizedExtensions.length > 0
+            ? normalizedExtensions.map((extension) => `.${extension}`).join(", ")
+            : "do tipo solicitado";
+        return {
+            total,
+            path: directoryPath,
+            extensions: normalizedExtensions,
+            recursive,
+            extensionsLabel,
+        };
+    }
     async runShellCommand(command, cwd) {
         if (!isSafeShellCommand(command)) {
             throw new Error("Nenhum comando shell foi informado para execucao local.");
@@ -1849,6 +2205,9 @@ if let output = String(data: data, encoding: .utf8) {
         if (action.type === "list_files") {
             return `Arquivos listados em ${action.path}`;
         }
+        if (action.type === "count_files") {
+            return `Arquivos contados em ${action.path}`;
+        }
         if (action.type === "run_shell") {
             return `Comando ${action.command} executado no macOS`;
         }
@@ -1858,6 +2217,9 @@ if let output = String(data: data, encoding: .utf8) {
         if (action.type === "click_visual_target") {
             return `Clique guiado executado para ${action.description}`;
         }
+        if (action.type === "drag_visual_target") {
+            return `Arraste guiado executado de ${action.source_description} para ${action.target_description}`;
+        }
         const target = humanizeUrl(action.url);
         return `${target} foi aberto${action.app ? ` em ${action.app}` : ""}`;
     }

package/dist/types.js

@@ -1,5 +1,5 @@
 export const BRIDGE_CONFIG_VERSION = 1;
-export const BRIDGE_VERSION = "0.5.5";
+export const BRIDGE_VERSION = "0.5.6";
 export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
 export const DEFAULT_API_BASE_URL = "http://localhost:8000";
 export const DEFAULT_POLL_INTERVAL_MS = 3000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leg3ndy/otto-bridge",
-  "version": "0.5.5",
+  "version": "0.5.6",
   "private": false,
   "type": "module",
   "description": "Local companion for Otto Bridge device pairing and WebSocket runtime.",