@leg3ndy/otto-bridge 0.5.0 → 0.5.1

package/dist/executors/native_macos.js

@@ -175,50 +175,7 @@ function isSafeShellCommand(command) {
     if (!trimmed) {
         return false;
     }
-    const forbiddenPatterns = [
-        /(^|[;&|])\s*sudo\b/i,
-        /\brm\b/i,
-        /\bmv\b/i,
-        /\bcp\b/i,
-        /\bchmod\b/i,
-        /\bchown\b/i,
-        /\bshutdown\b/i,
-        /\breboot\b/i,
-        /\bmkfs\b/i,
-        /\bdd\b/i,
-        /\bkill(?:all)?\b/i,
-        />/,
-        />>/,
-    ];
-    if (forbiddenPatterns.some((pattern) => pattern.test(trimmed))) {
-        return false;
-    }
-    const normalized = trimmed.replace(/\s+/g, " ");
-    const allowedPrefixes = [
-        "pwd",
-        "ls",
-        "cat ",
-        "cat",
-        "sed ",
-        "rg ",
-        "find ",
-        "git status",
-        "git log",
-        "git diff",
-        "head ",
-        "tail ",
-        "wc ",
-        "stat ",
-        "file ",
-        "mdls ",
-        "whoami",
-        "date",
-        "uname",
-        "python3 --version",
-        "node -v",
-        "npm -v",
-    ];
-    return allowedPrefixes.some((prefix) => normalized === prefix || normalized.startsWith(`${prefix} `));
+    return true;
 }
 function extractConfirmationOptions(job, actions) {
     const payload = asRecord(job.payload);
@@ -495,11 +452,13 @@ export class NativeMacOSJobExecutor {
                 if (action.type === "open_app") {
                     await reporter.progress(progressPercent, `Abrindo ${action.app} no macOS`);
                     await this.openApp(action.app);
+                    completionNotes.push(`${action.app} foi aberto no macOS.`);
                     continue;
                 }
                 if (action.type === "focus_app") {
                     await reporter.progress(progressPercent, `Trazendo ${action.app} para frente`);
                     await this.focusApp(action.app);
+                    completionNotes.push(`${action.app} ficou em foco no macOS.`);
                     continue;
                 }
                 if (action.type === "press_shortcut") {
@@ -521,8 +480,18 @@ export class NativeMacOSJobExecutor {
                 if (action.type === "take_screenshot") {
                     await reporter.progress(progressPercent, "Capturando screenshot do Mac");
                     const screenshotPath = await this.takeScreenshot(action.path);
-                    const screenshotArtifact = await this.uploadArtifactForJob(job.job_id, screenshotPath, {
+                    const uploadable = await this.buildUploadableImage(screenshotPath);
+                    const screenshotArtifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
                         kind: "screenshot",
+                        mimeTypeOverride: uploadable.mimeType,
+                        fileNameOverride: uploadable.filename,
+                        metadata: {
+                            width: uploadable.dimensions?.width || undefined,
+                            height: uploadable.dimensions?.height || undefined,
+                            original_width: uploadable.originalDimensions?.width || undefined,
+                            original_height: uploadable.originalDimensions?.height || undefined,
+                            resized_for_upload: uploadable.resized,
+                        },
                     });
                     if (screenshotArtifact) {
                         artifacts.push(screenshotArtifact);
@@ -537,6 +506,29 @@ export class NativeMacOSJobExecutor {
                 if (action.type === "read_frontmost_page") {
                     await reporter.progress(progressPercent, `Lendo a pagina ativa em ${action.app || "Safari"}`);
                     const page = await this.readFrontmostPage(action.app || "Safari");
+                    if (!page.text && this.bridgeConfig?.apiBaseUrl && this.bridgeConfig?.deviceToken) {
+                        await reporter.progress(progressPercent, "Safari bloqueou leitura direta; vou analisar a pagina pela tela");
+                        const screenshotPath = await this.takeScreenshot();
+                        const uploadable = await this.buildUploadableImage(screenshotPath);
+                        const artifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
+                            kind: "screenshot",
+                            mimeTypeOverride: uploadable.mimeType,
+                            fileNameOverride: uploadable.filename,
+                            metadata: {
+                                purpose: "page_read_fallback",
+                                width: uploadable.dimensions?.width || undefined,
+                                height: uploadable.dimensions?.height || undefined,
+                                original_width: uploadable.originalDimensions?.width || undefined,
+                                original_height: uploadable.originalDimensions?.height || undefined,
+                                resized_for_upload: uploadable.resized,
+                            },
+                        });
+                        if (artifact?.storage_path) {
+                            artifacts.push(artifact);
+                            const answer = await this.analyzeUploadedArtifact(job.job_id, artifact.storage_path, "Leia o que esta visivel nesta pagina da web e resuma em portugues brasileiro o conteudo principal. Inclua titulos, chamadas e o que parecer mais importante na tela.", artifact.mime_type);
+                            page.text = answer || page.text;
+                        }
+                    }
                     resultPayload.page = page;
                     completionNotes.push(`Li a pagina ${page.title || page.url || "ativa"} no navegador.`);
                     continue;
@@ -572,9 +564,20 @@ export class NativeMacOSJobExecutor {
                     }
                     await reporter.progress(progressPercent, `Capturando a tela para localizar ${action.description}`);
                     const screenshotPath = await this.takeScreenshot();
-                    const artifact = await this.uploadArtifactForJob(job.job_id, screenshotPath, {
+                    const uploadable = await this.buildUploadableImage(screenshotPath);
+                    const artifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
                         kind: "screenshot",
-                        metadata: { purpose: "visual_click", target: action.description },
+                        mimeTypeOverride: uploadable.mimeType,
+                        fileNameOverride: uploadable.filename,
+                        metadata: {
+                            purpose: "visual_click",
+                            target: action.description,
+                            width: uploadable.dimensions?.width || undefined,
+                            height: uploadable.dimensions?.height || undefined,
+                            original_width: uploadable.originalDimensions?.width || undefined,
+                            original_height: uploadable.originalDimensions?.height || undefined,
+                            resized_for_upload: uploadable.resized,
+                        },
                     });
                     if (!artifact?.storage_path) {
                         throw new Error("Otto Bridge nao conseguiu enviar a screenshot necessaria para localizar o alvo visual.");
@@ -583,19 +586,28 @@ export class NativeMacOSJobExecutor {
                     const artifactMetadata = artifact.metadata || {};
                     const width = Number(artifactMetadata.width || 0);
                     const height = Number(artifactMetadata.height || 0);
+                    const originalWidth = Number(artifactMetadata.original_width || width || 0);
+                    const originalHeight = Number(artifactMetadata.original_height || height || 0);
                     const location = await this.locateVisualTarget(job.job_id, artifact.storage_path, action.description, width, height, artifact.mime_type);
                     if (!location?.found || typeof location.x !== "number" || typeof location.y !== "number") {
                         throw new Error(`Nao consegui localizar ${action.description} com confianca suficiente na tela.`);
                     }
                     await reporter.progress(progressPercent, `Clicando em ${action.description}`);
-                    await this.clickPoint(location.x, location.y);
+                    const scaledX = width > 0 && originalWidth > 0 ? (location.x / width) * originalWidth : location.x;
+                    const scaledY = height > 0 && originalHeight > 0 ? (location.y / height) * originalHeight : location.y;
+                    await this.clickPoint(scaledX, scaledY);
                     completionNotes.push(`Localizei e cliquei em ${action.description}.`);
-                    resultPayload.last_click = location;
+                    resultPayload.last_click = {
+                        ...location,
+                        x: scaledX,
+                        y: scaledY,
+                    };
                     continue;
                 }
                 await reporter.progress(progressPercent, `Abrindo ${action.url}${action.app ? ` em ${action.app}` : ""}`);
                 await this.openUrl(action.url, action.app);
                 await delay(1200);
+                completionNotes.push(`${humanizeUrl(action.url)} foi aberto${action.app ? ` em ${action.app}` : ""}.`);
             }
             const summary = completionNotes.length > 0
                 ? completionNotes.join("\n\n")
@@ -713,8 +725,8 @@ end tell
             return null;
         }
         const bytes = await readFile(localPath);
-        const fileName = path.basename(localPath);
-        const mimeType = mimeTypeFromPath(fileName);
+        const fileName = options?.fileNameOverride || path.basename(localPath);
+        const mimeType = options?.mimeTypeOverride || mimeTypeFromPath(fileName);
         const dimensions = mimeType.startsWith("image/") ? await this.getImageDimensions(localPath) : null;
         const metadata = {
             ...(options?.metadata || {}),
@@ -729,6 +741,17 @@ end tell
         });
         return response.artifact || null;
     }
+    async analyzeUploadedArtifact(jobId, storagePath, question, mimeType) {
+        if (!this.bridgeConfig?.apiBaseUrl || !this.bridgeConfig?.deviceToken) {
+            return "";
+        }
+        const response = await postDeviceJson(this.bridgeConfig.apiBaseUrl, this.bridgeConfig.deviceToken, `/v1/devices/jobs/${encodeURIComponent(jobId)}/vision/analyze`, {
+            storage_path: storagePath,
+            question,
+            mime_type: mimeType || "image/jpeg",
+        });
+        return String(response.answer || "").trim();
+    }
     async readFrontmostPage(app) {
         const targetApp = app || "Safari";
         if (targetApp !== "Safari") {
@@ -743,13 +766,37 @@ tell application "Safari"
 end tell
 return pageJson
 `;
-        const { stdout } = await this.runCommandCapture("osascript", ["-e", script]);
-        const parsed = JSON.parse(stdout.trim() || "{}");
-        return {
-            title: asString(parsed.title) || "",
-            url: asString(parsed.url) || "",
-            text: asString(parsed.text) || "",
-        };
+        try {
+            const { stdout } = await this.runCommandCapture("osascript", ["-e", script]);
+            const parsed = JSON.parse(stdout.trim() || "{}");
+            return {
+                title: asString(parsed.title) || "",
+                url: asString(parsed.url) || "",
+                text: asString(parsed.text) || "",
+            };
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message : String(error);
+            if (!detail.toLowerCase().includes("allow javascript from apple events")) {
+                throw error;
+            }
+            const metadataScript = `
+tell application "Safari"
+  activate
+  if (count of windows) = 0 then error "Safari nao possui janelas abertas."
+  set pageTitle to name of current tab of front window
+  set pageUrl to URL of current tab of front window
+end tell
+return pageTitle & linefeed & pageUrl
+`;
+            const { stdout } = await this.runCommandCapture("osascript", ["-e", metadataScript]);
+            const [title, url] = stdout.split("\n");
+            return {
+                title: String(title || "").trim(),
+                url: String(url || "").trim(),
+                text: "",
+            };
+        }
     }
     async setVolume(level) {
         const bounded = Math.max(0, Math.min(Math.round(level), 100));
@@ -810,6 +857,56 @@ post(.leftMouseUp)
             return null;
         }
     }
+    async buildUploadableImage(localPath) {
+        const originalDimensions = await this.getImageDimensions(localPath);
+        const artifactsDir = path.join(os.homedir(), ".otto-bridge", "artifacts");
+        await mkdir(artifactsDir, { recursive: true });
+        let sourcePath = localPath;
+        let mimeType = mimeTypeFromPath(localPath);
+        let filename = path.basename(localPath);
+        let resized = false;
+        const conversionSteps = [
+            { width: 1280, quality: 42 },
+            { width: 1024, quality: 35 },
+            { width: 900, quality: 30 },
+            { width: 768, quality: 26 },
+            { width: 640, quality: 22 },
+            { width: 540, quality: 18 },
+            { width: 480, quality: 16 },
+        ];
+        for (const step of conversionSteps) {
+            const candidatePath = path.join(artifactsDir, `${path.basename(localPath, path.extname(localPath))}-${step.width}w-q${step.quality}.jpg`);
+            await this.runCommand("sips", [
+                "-s",
+                "format",
+                "jpeg",
+                "-s",
+                "formatOptions",
+                String(step.quality),
+                "--resampleWidth",
+                String(step.width),
+                localPath,
+                "--out",
+                candidatePath,
+            ]);
+            const candidateStat = await stat(candidatePath);
+            sourcePath = candidatePath;
+            mimeType = "image/jpeg";
+            filename = path.basename(candidatePath);
+            resized = true;
+            if (candidateStat.size <= 220_000) {
+                break;
+            }
+        }
+        return {
+            path: sourcePath,
+            mimeType,
+            filename,
+            dimensions: await this.getImageDimensions(sourcePath),
+            originalDimensions,
+            resized,
+        };
+    }
     async readLocalFile(filePath, maxChars = 4000) {
         const resolved = expandUserPath(filePath);
         const content = await readFile(resolved, "utf8");
@@ -834,7 +931,7 @@ post(.leftMouseUp)
     }
     async runShellCommand(command, cwd) {
         if (!isSafeShellCommand(command)) {
-            throw new Error("Otto Bridge permite apenas shell de consulta no momento. Use comandos de leitura como pwd, ls, cat, rg, find ou git status.");
+            throw new Error("Nenhum comando shell foi informado para execucao local.");
         }
         const resolvedCwd = cwd ? expandUserPath(cwd) : process.cwd();
         const { stdout, stderr } = await this.runCommandCapture("/bin/zsh", ["-lc", command], {
@@ -936,10 +1033,6 @@ post(.leftMouseUp)
             if (stderrText) {
                 console.warn(`[otto-bridge] ${command} stderr=${stderrText}`);
             }
-            const stdoutText = stdout.trim();
-            if (stdoutText) {
-                console.log(`[otto-bridge] ${command} stdout=${stdoutText}`);
-            }
             return { stdout, stderr };
         }
         catch (error) {

package/dist/types.js

@@ -1,5 +1,5 @@
 export const BRIDGE_CONFIG_VERSION = 1;
-export const BRIDGE_VERSION = "0.5.0";
+export const BRIDGE_VERSION = "0.5.1";
 export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
 export const DEFAULT_API_BASE_URL = "http://localhost:8000";
 export const DEFAULT_POLL_INTERVAL_MS = 3000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leg3ndy/otto-bridge",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "private": false,
   "type": "module",
   "description": "Local companion for Otto Bridge device pairing and WebSocket runtime.",