oh-my-opencode 4.12.1 → 4.14.0

package/packages/shared-skills/skills/ultimate-browsing/engine/templates/playwright_mobile_chrome.js

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+/**
+ * Generic Playwright mobile fetcher — real Chrome + device emulation.
+ *
+ * Usage:
+ *   echo '{"url":"...", "device":"iPhone 13 Pro"}' | node playwright_mobile_chrome.js
+ *
+ * Device name must match playwright `devices[...]` keys (Pixel 7, iPhone 13 Pro,
+ * iPad Pro 11, etc.). When in doubt, omit `device` — default is iPhone 13 Pro.
+ *
+ * NO-SITE-NAME RULE: same as playwright_real_chrome.js — no hostname branches.
+ */
+
+async function readStdinJson() {
+  return await new Promise((resolve, reject) => {
+    let data = '';
+    process.stdin.on('data', (c) => (data += c));
+    process.stdin.on('end', () => {
+      try { resolve(JSON.parse(data || '{}')); }
+      catch (e) { reject(e); }
+    });
+    process.stdin.on('error', reject);
+  });
+}
+
+function describeError(error) {
+  if (error instanceof Error) {
+    return `${error.name}: ${error.message}`;
+  }
+  return String(error);
+}
+
+function warnBestEffort(action, error) {
+  process.stderr.write(`best-effort ${action} failed: ${describeError(error)}\n`);
+}
+
+function isMissingTopLevelModule(error, moduleName) {
+  return (
+    error instanceof Error &&
+    error.code === 'MODULE_NOT_FOUND' &&
+    typeof error.message === 'string' &&
+    error.message.includes(`Cannot find module '${moduleName}'`)
+  );
+}
+
+function requireOptionalModule(moduleName) {
+  let resolvedModule;
+  try {
+    resolvedModule = require.resolve(moduleName);
+  } catch (e) {
+    if (isMissingTopLevelModule(e, moduleName)) {
+      warnBestEffort(`optional module ${moduleName}`, e);
+      return null;
+    }
+    throw e;
+  }
+  return require(resolvedModule);
+}
+
+async function main() {
+  const args = await readStdinJson();
+  const url = args.url;
+  if (!url) {
+    process.stderr.write('missing url\n');
+    process.exitCode = 2;
+    return;
+  }
+
+  const profileDir = args.profileDir || '/tmp/.insane_pw_mobile_profile';
+  const deviceName = args.device || 'iPhone 13 Pro';
+  const waitSelector = args.waitSelector || null;
+  const timeoutMs = args.timeout || 60000;
+  const headless = args.headless ?? false;
+
+  let chromium, devices;
+  const playwrightExtra = requireOptionalModule('playwright-extra');
+  const stealthPlugin = playwrightExtra ? requireOptionalModule('puppeteer-extra-plugin-stealth') : null;
+  if (playwrightExtra && stealthPlugin) {
+    ({ chromium, devices } = playwrightExtra);
+    const stealth = stealthPlugin();
+    chromium.use(stealth);
+  } else {
+    ({ chromium, devices } = require('playwright'));
+  }
+
+  const dev = devices[deviceName];
+  if (!dev) {
+    process.stderr.write(`unknown device: ${deviceName}\n`);
+    process.exitCode = 2;
+    return;
+  }
+
+  let ctx;
+  try {
+    ctx = await chromium.launchPersistentContext(profileDir, {
+      channel: 'chrome',
+      headless,
+      ...dev,
+    });
+    const page = await ctx.newPage();
+    const navTimeout = Math.min(timeoutMs, 90000);
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: navTimeout });
+
+    if (waitSelector) {
+      try {
+        await page.waitForSelector(waitSelector, { timeout: Math.min(timeoutMs, 20000) });
+      } catch (e) {
+        warnBestEffort('waitSelector', e);
+      }
+    }
+
+    const html = await page.content();
+    process.stdout.write(html);
+    process.exitCode = 0;
+    return;
+  } catch (e) {
+    process.stderr.write(`${describeError(e)}\n`);
+    process.exitCode = 1;
+    return;
+  } finally {
+    try {
+      if (ctx) await ctx.close();
+    } catch (e) {
+      warnBestEffort('browser context close', e);
+    }
+  }
+}
+
+main();

package/packages/shared-skills/skills/ultimate-browsing/engine/templates/playwright_real_chrome.js

@@ -0,0 +1,167 @@
+#!/usr/bin/env node
+/**
+ * Generic Playwright fetcher — real Chrome channel (not bundled Chromium).
+ *
+ * Usage (driven by engine/executor.py):
+ *   echo '{"url":"...", "profileDir":"/tmp/.p", "waitSelector":"article"}' | node playwright_real_chrome.js
+ *
+ * Outputs page HTML to stdout on success; errors to stderr with non-zero exit.
+ *
+ * NO-SITE-NAME RULE: this file must never branch on specific hostnames.
+ * All site specifics come from the JSON input (url, waitSelector).
+ *
+ * Dependencies (install once on target machine):
+ *   npm i -g playwright playwright-extra puppeteer-extra-plugin-stealth
+ *   npx playwright install chrome    # system Chrome binary
+ */
+
+const fs = require('fs');
+
+async function readStdinJson() {
+  return await new Promise((resolve, reject) => {
+    let data = '';
+    process.stdin.on('data', (c) => (data += c));
+    process.stdin.on('end', () => {
+      try { resolve(JSON.parse(data || '{}')); }
+      catch (e) { reject(e); }
+    });
+    process.stdin.on('error', reject);
+  });
+}
+
+function describeError(error) {
+  if (error instanceof Error) {
+    return `${error.name}: ${error.message}`;
+  }
+  return String(error);
+}
+
+function warnBestEffort(action, error) {
+  process.stderr.write(`best-effort ${action} failed: ${describeError(error)}\n`);
+}
+
+function isMissingTopLevelModule(error, moduleName) {
+  return (
+    error instanceof Error &&
+    error.code === 'MODULE_NOT_FOUND' &&
+    typeof error.message === 'string' &&
+    error.message.includes(`Cannot find module '${moduleName}'`)
+  );
+}
+
+function requireOptionalModule(moduleName) {
+  let resolvedModule;
+  try {
+    resolvedModule = require.resolve(moduleName);
+  } catch (e) {
+    if (isMissingTopLevelModule(e, moduleName)) {
+      warnBestEffort(`optional module ${moduleName}`, e);
+      return null;
+    }
+    throw e;
+  }
+  return require(resolvedModule);
+}
+
+async function main() {
+  const args = await readStdinJson();
+  const url = args.url;
+  if (!url) {
+    process.stderr.write('missing url\n');
+    process.exitCode = 2;
+    return;
+  }
+
+  const profileDir = args.profileDir || '/tmp/.insane_pw_profile';
+  const waitSelector = args.waitSelector || null;
+  const timeoutMs = args.timeout || 60000;
+  const headless = args.headless ?? false;     // Akamai/etc detect headless
+  const viewport = args.viewport || { width: 1366, height: 900 };
+
+  let chromium;
+  const playwrightExtra = requireOptionalModule('playwright-extra');
+  const stealthPlugin = playwrightExtra ? requireOptionalModule('puppeteer-extra-plugin-stealth') : null;
+  if (playwrightExtra && stealthPlugin) {
+    ({ chromium } = playwrightExtra);
+    const stealth = stealthPlugin();
+    chromium.use(stealth);
+  } else {
+    // Fallback to plain playwright (no stealth). Still uses channel:chrome.
+    ({ chromium } = require('playwright'));
+  }
+
+  let ctx;
+  try {
+    ctx = await chromium.launchPersistentContext(profileDir, {
+      channel: 'chrome',          // real Chrome, not bundled Chromium
+      headless,
+      viewport,
+    });
+    const page = await ctx.newPage();
+    const navTimeout = Math.min(timeoutMs, 90000);
+
+    // Warmup hop: visit the site root first so Akamai-style bot managers
+    // can run their JS sensor and set a resolved session cookie. Direct
+    // landing on a search/deep URL is the classic first-hit rejection pattern.
+    // Use domcontentloaded (not networkidle) — many SPAs keep analytics/xhr
+    // open indefinitely and would hit the 90s timeout.
+    try {
+      const urlObj = new URL(url);
+      const rootUrl = `${urlObj.protocol}//${urlObj.host}/`;
+      if (rootUrl !== url) {
+        await page.goto(rootUrl, { waitUntil: 'domcontentloaded', timeout: navTimeout });
+        await page.waitForTimeout(3500);   // let sensor JS finish
+      }
+    } catch (e) {
+      warnBestEffort('warmup navigation', e);
+      // warmup is best-effort; continue even if it hiccups
+    }
+
+    // Main page — DOM loaded then give the sensor a moment.
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: navTimeout });
+    await page.waitForTimeout(2500);
+
+    if (waitSelector) {
+      try {
+        await page.waitForSelector(waitSelector, { timeout: Math.min(timeoutMs, 20000) });
+      } catch (e) {
+        warnBestEffort('waitSelector', e);
+        // Selector still missing — try one hard reload in case the first hit
+        // landed on a challenge page and the sensor has just cleared.
+        try {
+          await page.reload({ waitUntil: 'domcontentloaded', timeout: navTimeout });
+          await page.waitForTimeout(2000);
+          try {
+            await page.waitForSelector(waitSelector, { timeout: 10000 });
+          } catch (e2) {
+            warnBestEffort('retry waitSelector', e2);
+            // Still no luck — caller validates HTML anyway.
+          }
+        } catch (e3) {
+          warnBestEffort('selector recovery reload', e3);
+          // reload failed — proceed with whatever we have
+        }
+      }
+    } else {
+      // Without a positive-proof selector, give the sensor a couple more seconds.
+      await page.waitForTimeout(2000);
+    }
+
+    const html = await page.content();
+    process.stdout.write(html);
+    process.exitCode = 0;
+    return;
+  } catch (e) {
+    process.stderr.write(`${describeError(e)}\n`);
+    process.exitCode = 1;
+    return;
+  } finally {
+    try {
+      if (ctx) await ctx.close();
+    } catch (e) {
+      warnBestEffort('browser context close', e);
+    }
+  }
+}
+
+main();

package/packages/shared-skills/skills/ultimate-browsing/engine/tests/test_fetch_chain.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from engine.fetch_chain import fetch  # noqa: E402
+from engine.result_schema import Attempt  # noqa: E402
+from engine.validators import Verdict  # noqa: E402
+
+
+class _Resp:
+    def __init__(self, text: str = "<article>ok</article>", url: str = "https://example.com/"):
+        self.text = text
+        self.url = url
+
+
+class _Hit:
+    profile_id = "cloudflare_turnstile"
+    confidence = 1.0
+    signals = ["test"]
+
+
+class FetchChain(unittest.TestCase):
+    def test_probe_success_returns_without_grid(self) -> None:
+        attempt = Attempt(
+            phase="probe",
+            executor="curl_cffi",
+            url="https://example.com",
+            url_transform="original",
+            impersonate="safari",
+            referer="self_root",
+            verdict=Verdict.WEAK_OK.value,
+        )
+
+        with patch("engine.fetch_chain._load_profiles", return_value={}), \
+                patch("engine.fetch_chain.last_load_error", return_value=None), \
+                patch("engine.fetch_chain.run_attempt", return_value=(attempt, _Resp())) as run_attempt:
+            result = fetch("https://example.com", enable_playwright=False)
+
+        self.assertTrue(result.ok)
+        self.assertEqual(result.verdict, Verdict.WEAK_OK.value)
+        self.assertEqual(len(result.trace), 1)
+        self.assertEqual(run_attempt.call_count, 1)
+
+    def test_max_attempts_stops_after_probe_before_grid(self) -> None:
+        attempt = Attempt(
+            phase="probe",
+            executor="curl_cffi",
+            url="https://example.com",
+            url_transform="original",
+            impersonate="safari",
+            referer="self_root",
+            verdict=Verdict.CHALLENGE.value,
+        )
+
+        with patch("engine.fetch_chain._load_profiles", return_value={}), \
+                patch("engine.fetch_chain.last_load_error", return_value=None), \
+                patch("engine.fetch_chain.run_attempt", return_value=(attempt, _Resp("blocked"))), \
+                patch("engine.fetch_chain.detect", return_value=[_Hit()]), \
+                patch("engine.fetch_chain.load_profile", return_value={
+                    "tls_impersonate_candidates": [["chrome"]],
+                    "referer_strategies": ["self_root"],
+                    "url_transform_order": ["original"],
+                }):
+            result = fetch("https://example.com", max_attempts=1, enable_playwright=False)
+
+        self.assertFalse(result.ok)
+        self.assertEqual(len(result.trace), 1)
+        self.assertEqual(result.trace[0].phase, "probe")
+
+
+if __name__ == "__main__":
+    unittest.main()

package/packages/shared-skills/skills/ultimate-browsing/engine/tests/test_playwright_templates.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import textwrap
+import unittest
+from pathlib import Path
+
+TEMPLATES_DIR = Path(__file__).resolve().parents[1] / "templates"
+TEMPLATE_NAMES = ("playwright_real_chrome.js", "playwright_mobile_chrome.js")
+JsonValue = None | bool | int | float | str | list["JsonValue"] | dict[str, "JsonValue"]
+
+
+def _write_file(path: Path, content: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(textwrap.dedent(content).lstrip(), encoding="utf-8")
+
+
+def _install_fake_playwright(node_modules: Path) -> None:
+    _write_file(
+        node_modules / "playwright" / "index.js",
+        """
+        const page = {
+          async goto() {},
+          async waitForTimeout() {},
+          async waitForSelector(selector) {
+            if (process.env.PW_FAKE_SELECTOR_FAIL === '1') {
+              throw new Error(`missing selector ${selector}`);
+            }
+          },
+          async reload() {},
+          async content() {
+            return '<html><article>ok</article></html>';
+          },
+        };
+
+        const context = {
+          async newPage() { return page; },
+          async close() {
+            if (process.env.PW_FAKE_CLOSE_FAIL === '1') {
+              throw new Error('close failed');
+            }
+          },
+        };
+
+        exports.chromium = {
+          use() {},
+          async launchPersistentContext() { return context; },
+        };
+        exports.devices = {
+          'iPhone 13 Pro': { viewport: { width: 390, height: 844 }, userAgent: 'fake-mobile' },
+        };
+        """,
+    )
+
+
+def _install_broken_playwright_extra(node_modules: Path) -> None:
+    _write_file(
+        node_modules / "playwright-extra" / "index.js",
+        """
+        const error = new Error('stealth init failed');
+        error.code = 'EACCES';
+        throw error;
+        """,
+    )
+    _write_file(
+        node_modules / "puppeteer-extra-plugin-stealth" / "index.js",
+        """
+        module.exports = function stealth() { return {}; };
+        """,
+    )
+
+
+def _install_working_playwright_extra(node_modules: Path) -> None:
+    _write_file(
+        node_modules / "playwright-extra" / "index.js",
+        """
+        const playwright = require('playwright');
+        exports.chromium = playwright.chromium;
+        exports.devices = playwright.devices;
+        """,
+    )
+
+
+def _install_playwright_extra_with_internal_missing_dependency(node_modules: Path) -> None:
+    _write_file(
+        node_modules / "playwright-extra" / "index.js",
+        """
+        require('transitive-stealth-runtime');
+        """,
+    )
+
+
+def _install_self_missing_optional_module(node_modules: Path, module_name: str) -> None:
+    _write_file(
+        node_modules / module_name / "index.js",
+        f"""
+        const error = new Error("Cannot find module '{module_name}'");
+        error.code = 'MODULE_NOT_FOUND';
+        throw error;
+        """,
+    )
+
+
+def _run_template(
+    template_name: str,
+    payload: dict[str, JsonValue],
+    *,
+    include_broken_extra: bool = False,
+    include_working_extra: bool = False,
+    include_internal_missing_extra: bool = False,
+    include_self_missing_module: str | None = None,
+    env_overrides: dict[str, str] | None = None,
+) -> subprocess.CompletedProcess[str]:
+    with tempfile.TemporaryDirectory(prefix="ultimate-browsing-template-test-") as tmp:
+        tmp_path = Path(tmp)
+        node_modules = tmp_path / "node_modules"
+        _install_fake_playwright(node_modules)
+        if include_working_extra:
+            _install_working_playwright_extra(node_modules)
+        if include_broken_extra:
+            _install_broken_playwright_extra(node_modules)
+        if include_internal_missing_extra:
+            _install_playwright_extra_with_internal_missing_dependency(node_modules)
+        if include_self_missing_module:
+            _install_self_missing_optional_module(node_modules, include_self_missing_module)
+
+        script_path = tmp_path / template_name
+        script_path.write_text((TEMPLATES_DIR / template_name).read_text(encoding="utf-8"), encoding="utf-8")
+
+        env = os.environ.copy()
+        env.pop("NODE_PATH", None)
+        if env_overrides:
+            env.update(env_overrides)
+
+        return subprocess.run(
+            ["node", str(script_path)],
+            input=json.dumps(payload),
+            text=True,
+            capture_output=True,
+            timeout=5,
+            env=env,
+            check=False,
+        )
+
+
+@unittest.skipUnless(shutil.which("node"), "node is required for Playwright template tests")
+class PlaywrightTemplateErrorHandling(unittest.TestCase):
+    def test_missing_playwright_extra_warns_and_falls_back_to_plain_playwright(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                    },
+                )
+
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertIn("<article>ok</article>", result.stdout)
+                self.assertIn("best-effort optional module playwright-extra failed:", result.stderr)
+                self.assertIn("Cannot find module 'playwright-extra'", result.stderr)
+
+    def test_missing_stealth_plugin_warns_and_falls_back_to_plain_playwright(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                    },
+                    include_working_extra=True,
+                )
+
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertIn("<article>ok</article>", result.stdout)
+                self.assertIn("best-effort optional module puppeteer-extra-plugin-stealth failed:", result.stderr)
+                self.assertIn("Cannot find module 'puppeteer-extra-plugin-stealth'", result.stderr)
+
+    def test_non_missing_stealth_dependency_error_is_not_swallowed(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                    },
+                    include_broken_extra=True,
+                )
+
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn("stealth init failed", result.stderr)
+                self.assertNotIn("best-effort optional module", result.stderr)
+                self.assertEqual(result.stdout, "")
+
+    def test_internal_module_resolution_error_is_not_treated_as_optional_missing_dependency(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                    },
+                    include_internal_missing_extra=True,
+                )
+
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn("Cannot find module 'transitive-stealth-runtime'", result.stderr)
+                self.assertNotIn("best-effort optional module", result.stderr)
+                self.assertEqual(result.stdout, "")
+
+    def test_present_optional_module_self_missing_error_is_not_swallowed(self) -> None:
+        cases = (("playwright-extra", False), ("puppeteer-extra-plugin-stealth", True))
+        for template_name in TEMPLATE_NAMES:
+            for module_name, include_working_extra in cases:
+                with self.subTest(template_name=template_name, module_name=module_name):
+                    result = _run_template(
+                        template_name,
+                        {
+                            "url": "https://example.com/article",
+                            "profileDir": "/tmp/ultimate-browsing-test-profile",
+                            "headless": True,
+                        },
+                        include_working_extra=include_working_extra,
+                        include_self_missing_module=module_name,
+                    )
+
+                    self.assertNotEqual(result.returncode, 0)
+                    self.assertIn(f"Cannot find module '{module_name}'", result.stderr)
+                    self.assertNotIn("best-effort optional module", result.stderr)
+                    self.assertEqual(result.stdout, "")
+
+    def test_selector_failures_are_reported_as_best_effort_warnings(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                        "waitSelector": "article.ready",
+                    },
+                    env_overrides={"PW_FAKE_SELECTOR_FAIL": "1"},
+                )
+
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertIn("<article>ok</article>", result.stdout)
+                self.assertIn("best-effort waitSelector failed:", result.stderr)
+                self.assertNotIn("waitSelector article.ready", result.stderr)
+
+    def test_context_close_failures_are_reported_after_successful_html_output(self) -> None:
+        for template_name in TEMPLATE_NAMES:
+            with self.subTest(template_name=template_name):
+                result = _run_template(
+                    template_name,
+                    {
+                        "url": "https://example.com/article",
+                        "profileDir": "/tmp/ultimate-browsing-test-profile",
+                        "headless": True,
+                    },
+                    env_overrides={"PW_FAKE_CLOSE_FAIL": "1"},
+                )
+
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertIn("<article>ok</article>", result.stdout)
+                self.assertIn("best-effort browser context close failed:", result.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()