omnius 1.0.855 → 1.0.856

package/dist/scripts/crawlee-scraper.py

@@ -21,6 +21,9 @@ import json
 import subprocess
 import sys
 import importlib
+import re
+from datetime import timedelta
+from urllib.parse import urljoin, urlparse
 from typing import Any
 
 # ---------------------------------------------------------------------------
@@ -86,182 +89,160 @@ def ensure_playwright():
 
 
 # ---------------------------------------------------------------------------
-# BeautifulSoup crawler
+# Complete page observations, with explicit resource failures (never prefixes)
 # ---------------------------------------------------------------------------
-
-async def crawl_beautifulsoup(
-    start_url: str,
-    max_pages: int = 5,
-    max_depth: int = 1,
-    extract: str = "all",
-) -> list[dict[str, Any]]:
-    """Crawl using BeautifulSoup (HTTP-based, fast, no JS rendering)."""
-    from crawlee.beautifulsoup_crawler import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
-    from crawlee import ConcurrencySettings
-
-    results: list[dict[str, Any]] = []
-    pages_crawled = 0
-
-    crawler = BeautifulSoupCrawler(
-        max_request_retries=2,
-        request_handler_timeout=asyncio.timedelta(seconds=30),
-        max_requests_per_crawl=max_pages,
-        concurrency_settings=ConcurrencySettings(
-            max_concurrency=3,
-        ),
-    )
-
-    @crawler.router.default_handler
-    async def handler(context: BeautifulSoupCrawlingContext) -> None:
-        nonlocal pages_crawled
-        if pages_crawled >= max_pages:
+MAX_PAGE_BYTES = 5 * 1024 * 1024
+MAX_RESULT_BYTES = 8 * 1024 * 1024
+DESTINATION_ATTRIBUTES = ("href", "src", "srcset", "action", "formaction", "poster", "data-src", "data-srcset", "data-original")
+
+
+def extract_page(soup: Any, url: str) -> dict[str, Any]:
+    """Retain the entire parser/DOM snapshot and clean complete body view.
+
+    Snapshot provenance is deliberately not an HTTP-status or full-site claim.
+    Script/style/template payloads stay in the retained snapshot; JSON-LD is
+    additionally exposed without discarding malformed or duplicate entries.
+    """
+    from bs4 import Comment, NavigableString, Tag
+    snapshot = str(soup)
+    if len(snapshot.encode("utf-8")) > MAX_PAGE_BYTES:
+        raise ValueError("Page snapshot exceeded the complete-page resource limit; no prefix was retained")
+    body = soup.body or soup
+    links = [{"url": urljoin(url, str(tag.get("href"))), "text": tag.get_text(" ", strip=True)}
+             for tag in soup.find_all("a", href=True)]
+    metadata = [{"name": str(tag.get("name") or tag.get("property") or tag.get("http-equiv") or ""),
+                 "content": str(tag.get("content") or "")} for tag in soup.find_all("meta")]
+    structured = []
+    for tag in soup.find_all("script"):
+        if "ld+json" in str(tag.get("type", "")).lower():
+            payload = tag.string if tag.string is not None else tag.get_text()
+            try:
+                structured.append({"type": str(tag.get("type")), "value": json.loads(payload), "raw": payload})
+            except (ValueError, TypeError):
+                structured.append({"type": str(tag.get("type")), "raw": payload, "parse_error": True})
+
+    parts: list[str] = []
+    def visit(node: Any, preformatted: bool = False) -> None:
+        if isinstance(node, Comment):
             return
-        pages_crawled += 1
-
-        soup = context.soup
-        page_data: dict[str, Any] = {
-            "url": context.request.url,
-            "status": 200,
-        }
-
-        # Extract title
-        title_tag = soup.find("title")
-        page_data["title"] = title_tag.get_text(strip=True) if title_tag else ""
-
-        # Extract text content
-        if extract in ("text", "all"):
-            # Remove script and style elements
-            for tag in soup(["script", "style", "nav", "footer", "header"]):
-                tag.decompose()
-            text = soup.get_text(separator="\n", strip=True)
-            # Collapse whitespace
-            lines = [line.strip() for line in text.splitlines() if line.strip()]
-            page_data["text"] = "\n".join(lines)
-
-        # Extract links
-        if extract in ("links", "all"):
-            links = []
-            for a_tag in soup.find_all("a", href=True):
-                href = str(a_tag["href"])
-                link_text = a_tag.get_text(strip=True)
-                if href.startswith(("http://", "https://")):
-                    links.append({"url": href, "text": link_text})
-            page_data["links"] = links[:50]  # Cap at 50 links per page
-
-        # Extract metadata
-        meta_tags = {}
-        for meta in soup.find_all("meta"):
-            name = meta.get("name", meta.get("property", ""))
-            content = meta.get("content", "")
-            if name and content:
-                meta_tags[str(name)] = str(content)
-        if meta_tags:
-            page_data["meta"] = meta_tags
-
-        results.append(page_data)
-
-        # Enqueue links for deeper crawling if depth allows
-        if max_depth > 0:
-            await context.enqueue_links(strategy="same-domain")
-
-    await crawler.run([start_url])
+        if isinstance(node, NavigableString):
+            # Preserve code/pre whitespace exactly; normal HTML whitespace is
+            # collapsed without removing navigation/footer or document tails.
+            parts.append(str(node) if preformatted else re.sub(r"\s+", " ", str(node)))
+            return
+        if not isinstance(node, Tag):
+            return
+        for attribute in DESTINATION_ATTRIBUTES:
+            value = node.get(attribute)
+            if value is not None:
+                parts.append("\n[" + attribute + "=" + str(value) + "]\n")
+        if node.name == "img" and node.get("alt") is not None:
+            parts.append(str(node.get("alt")))
+        if node.name in ("script", "style", "template"):
+            return
+        block = node.name in ("address", "article", "aside", "blockquote", "br", "caption", "div", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "p", "pre", "section", "table", "tr")
+        if block:
+            parts.append("\n")
+        for child in node.children:
+            visit(child, preformatted or node.name in ("pre", "code"))
+            if node.name in ("td", "th"):
+                parts.append("\t")
+        if block:
+            parts.append("\n")
+    visit(body)
+    title = soup.find("title")
+    return {"url": url, "title": title.get_text(" ", strip=True) if title else "", "text": "".join(parts),
+            "html_snapshot": snapshot, "links": links, "metadata": metadata, "structured_data": structured,
+            "text_coverage": {"scope": "all_static_body_text", "rendered": False,
+                              "excluded_payload_elements": ["script", "style", "template"],
+                              "included_attributes": list(DESTINATION_ATTRIBUTES)}}
+
+
+def checked_results(results: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    if len(json.dumps(results, ensure_ascii=False).encode("utf-8")) > MAX_RESULT_BYTES:
+        raise ValueError("Crawl exceeded the complete-result resource limit; no partial success was retained")
     return results
 
 
-# ---------------------------------------------------------------------------
-# Playwright crawler
-# ---------------------------------------------------------------------------
-
-async def crawl_playwright(
-    start_url: str,
-    max_pages: int = 5,
-    max_depth: int = 1,
-    extract: str = "all",
-) -> list[dict[str, Any]]:
-    """Crawl using Playwright (headless browser, JS rendering)."""
-    from crawlee.playwright_crawler import PlaywrightCrawler, PlaywrightCrawlingContext
+async def crawl_pages(start_url: str, max_pages: int, max_depth: int, rendered: bool) -> list[dict[str, Any]]:
     from crawlee import ConcurrencySettings
-
+    if rendered:
+        from crawlee.playwright_crawler import PlaywrightCrawler as Crawler
+    else:
+        from crawlee.beautifulsoup_crawler import BeautifulSoupCrawler as Crawler
     results: list[dict[str, Any]] = []
-    pages_crawled = 0
-
-    crawler = PlaywrightCrawler(
-        max_request_retries=2,
-        request_handler_timeout=asyncio.timedelta(seconds=60),
-        max_requests_per_crawl=max_pages,
-        headless=True,
-        browser_type="chromium",
-        concurrency_settings=ConcurrencySettings(
-            max_concurrency=2,
-        ),
-    )
+    visited: set[str] = set()
+    frontier = [start_url]
+    domain = urlparse(start_url).netloc
+    # Breadth-first explicit levels avoid relying on crawler-specific private
+    # request depth fields. max_pages and max_depth define requested scope.
+    for depth in range(max_depth + 1):
+        urls = [url for url in frontier if url not in visited][:max_pages - len(results)]
+        if not urls:
+            break
+        failures: list[str] = []
+        crawler_options: dict[str, Any] = {
+            "max_request_retries": 2,
+            "request_handler_timeout": timedelta(seconds=60 if rendered else 30),
+            "max_requests_per_crawl": len(urls),
+            "concurrency_settings": ConcurrencySettings(max_concurrency=1),
+        }
+        if rendered:
+            crawler_options.update({"headless": True, "browser_type": "chromium"})
+        crawler = Crawler(**crawler_options)
+
+        @crawler.router.default_handler
+        async def handler(context: Any) -> None:
+            url = context.request.url
+            if rendered:
+                from bs4 import BeautifulSoup
+                await context.page.wait_for_load_state("domcontentloaded", timeout=15000)
+                soup = BeautifulSoup(await context.page.content(), "html.parser")
+            else:
+                soup = context.soup
+            page = extract_page(soup, url)
+            page["depth"] = depth
+            page["text_coverage"]["rendered"] = rendered
+            # Record an observed status only when the actual response exposes it.
+            response = getattr(context, "response", None) if rendered else getattr(context, "http_response", None)
+            status = getattr(response, "status", None) if rendered else getattr(response, "status_code", None)
+            if isinstance(status, int):
+                if status >= 400:
+                    raise RuntimeError(f"Page request failed with observed HTTP {status}")
+                page["observed_status"] = status
+            results.append(page)
+            checked_results(results)
+
+        @crawler.failed_request_handler
+        async def failed(context: Any, error: Exception) -> None:
+            failures.append(str(context.request.url))
+
+        await crawler.run(urls)
+        if failures or len(results) < len(visited) + len(urls):
+            raise RuntimeError("Crawl did not retrieve all dispatched pages; no partial success was retained")
+        visited.update(urls)
+        if len(results) >= max_pages:
+            break
+        frontier = []
+        seen = set(visited)
+        for page in results:
+            if page["depth"] != depth:
+                continue
+            for link in page["links"]:
+                candidate = link["url"]
+                parsed = urlparse(candidate)
+                if parsed.scheme in ("http", "https") and parsed.netloc == domain and candidate not in seen:
+                    frontier.append(candidate)
+                    seen.add(candidate)
+    return checked_results(results)
 
-    @crawler.router.default_handler
-    async def handler(context: PlaywrightCrawlingContext) -> None:
-        nonlocal pages_crawled
-        if pages_crawled >= max_pages:
-            return
-        pages_crawled += 1
 
-        page = context.page
-        page_data: dict[str, Any] = {
-            "url": context.request.url,
-            "status": 200,
-        }
+async def crawl_beautifulsoup(start_url: str, max_pages: int = 5, max_depth: int = 1, extract: str = "all") -> list[dict[str, Any]]:
+    return await crawl_pages(start_url, max_pages, max_depth, False)
 
-        # Wait for content to load
-        try:
-            await page.wait_for_load_state("networkidle", timeout=15000)
-        except Exception:
-            await page.wait_for_load_state("domcontentloaded", timeout=10000)
-
-        # Extract title
-        page_data["title"] = await page.title()
-
-        # Extract text content
-        if extract in ("text", "all"):
-            # Remove non-content elements via JS
-            text = await page.evaluate("""() => {
-                const remove = document.querySelectorAll('script, style, nav, footer, header, [role="navigation"]');
-                remove.forEach(el => el.remove());
-                return document.body ? document.body.innerText : '';
-            }""")
-            lines = [line.strip() for line in str(text).splitlines() if line.strip()]
-            page_data["text"] = "\n".join(lines)
-
-        # Extract links
-        if extract in ("links", "all"):
-            links = await page.evaluate("""() => {
-                const anchors = document.querySelectorAll('a[href]');
-                return Array.from(anchors).slice(0, 50).map(a => ({
-                    url: a.href,
-                    text: a.innerText.trim()
-                })).filter(l => l.url.startsWith('http'));
-            }""")
-            page_data["links"] = links
-
-        # Extract meta tags
-        meta_tags = await page.evaluate("""() => {
-            const metas = document.querySelectorAll('meta[name], meta[property]');
-            const result = {};
-            metas.forEach(m => {
-                const key = m.getAttribute('name') || m.getAttribute('property');
-                const val = m.getAttribute('content');
-                if (key && val) result[key] = val;
-            });
-            return result;
-        }""")
-        if meta_tags:
-            page_data["meta"] = meta_tags
-
-        results.append(page_data)
-
-        # Enqueue links for deeper crawling
-        if max_depth > 0:
-            await context.enqueue_links(strategy="same-domain")
-
-    await crawler.run([start_url])
-    return results
+
+async def crawl_playwright(start_url: str, max_pages: int = 5, max_depth: int = 1, extract: str = "all") -> list[dict[str, Any]]:
+    return await crawl_pages(start_url, max_pages, max_depth, True)
 
 
 # ---------------------------------------------------------------------------
@@ -297,6 +278,8 @@ async def main():
     )
 
     args = parser.parse_args()
+    if not 1 <= args.max_pages <= 50 or not 0 <= args.max_depth <= 5:
+        parser.error("max-pages must be 1..50 and max-depth must be 0..5")
 
     # Ensure crawlee is installed
     if not ensure_crawlee():
@@ -332,26 +315,23 @@ async def main():
         print(json.dumps(result))
         sys.exit(1)
 
-    # Output results
+    # Both display formats retain the complete structured source, including
+    # metadata and JSON-LD. Refuse an oversized result instead of a prefix.
+    output = {
+        "success": True,
+        "strategy": args.strategy,
+        "content_complete": True,
+        "requested_scope": {"max_pages": args.max_pages, "max_depth": args.max_depth},
+        "pages_crawled": len(results),
+        "results": results,
+    }
+    serialized = json.dumps(output, ensure_ascii=False, indent=2)
+    if len(serialized.encode("utf-8")) > MAX_RESULT_BYTES:
+        print(json.dumps({"error": "Crawl exceeded the complete-result resource limit; no partial success was retained"}))
+        sys.exit(1)
     if args.output == "text":
-        for page in results:
-            print(f"=== {page.get('title', 'Untitled')} ===")
-            print(f"URL: {page.get('url', '')}")
-            if "text" in page:
-                print(page["text"][:5000])
-            if "links" in page:
-                print(f"\nLinks ({len(page['links'])}):")
-                for link in page["links"][:20]:
-                    print(f"  - {link.get('text', '')}: {link.get('url', '')}")
-            print()
-    else:
-        output = {
-            "success": True,
-            "strategy": args.strategy,
-            "pages_crawled": len(results),
-            "results": results,
-        }
-        print(json.dumps(output, ensure_ascii=False, indent=2))
+        print("Complete structured crawl source:")
+    print(serialized)
 
 
 if __name__ == "__main__":

package/dist/scripts/web_scrape.py

@@ -184,8 +184,6 @@ def _serialize_script_result(value, depth: int = 0, seen: Optional[set[int]] = N
         seen = set()
     if value is None or isinstance(value, (str, int, float, bool)):
         return value
-    if depth > 5:
-        return str(value)
     if isinstance(value, WebElement):
         try:
             rect = value.rect or {}
@@ -212,7 +210,8 @@ def _serialize_script_result(value, depth: int = 0, seen: Optional[set[int]] = N
             "type": attr("type"),
             "role": attr("role"),
             "ariaLabel": attr("aria-label"),
-            "text": text[:240],
+            "text": text,
+            "__omnius_observation_scope": "element_descriptor",
             "rect": {
                 "x": rect.get("x", 0),
                 "y": rect.get("y", 0),
@@ -220,21 +219,25 @@ def _serialize_script_result(value, depth: int = 0, seen: Optional[set[int]] = N
                 "height": rect.get("height", 0),
             },
         }
-    if isinstance(value, (list, tuple, set)):
-        return [_serialize_script_result(item, depth + 1, seen) for item in list(value)[:200]]
+    if isinstance(value, (list, tuple)):
+        ident = id(value)
+        if ident in seen:
+            raise ValueError("Circular evaluation result is not JSON serializable")
+        seen.add(ident)
+        try:
+            return [_serialize_script_result(item, depth + 1, seen) for item in value]
+        finally:
+            seen.remove(ident)
     if isinstance(value, dict):
         ident = id(value)
         if ident in seen:
-            return "[Circular]"
+            raise ValueError("Circular evaluation result is not JSON serializable")
         seen.add(ident)
-        out = {}
-        for idx, (key, item) in enumerate(value.items()):
-            if idx >= 200:
-                out["__omnius_truncated"] = True
-                break
-            out[str(key)] = _serialize_script_result(item, depth + 1, seen)
-        return out
-    return str(value)
+        try:
+            return {str(key): _serialize_script_result(item, depth + 1, seen) for key, item in value.items()}
+        finally:
+            seen.remove(ident)
+    raise ValueError(f"Unsupported evaluation result type: {type(value).__name__}")
 
 
 class Tools:
@@ -486,13 +489,11 @@ class Tools:
             return f"Error typing into {selector}: {e}"
 
     @staticmethod
-    def get_dom_snapshot(max_chars: int = 200_000) -> str:
+    def get_dom_snapshot() -> str:
         if not Tools._driver:
             return ""
         try:
             dom = Tools._driver.execute_script("return document.documentElement.outerHTML;")
-            if dom and len(dom) > max_chars:
-                dom = dom[:max_chars]
             return dom or ""
         except Exception as exc:
             log_message(f"[dom] snapshot failed: {exc}", "WARNING")
@@ -508,6 +509,7 @@ class Tools:
                 "ok": True,
                 "result": _serialize_script_result(result),
                 "result_type": "undefined" if result is None else type(result).__name__,
+                "url": Tools._driver.current_url,
             }
         except Exception as exc:
             log_message(f"[evaluate] script failed: {exc}", "ERROR")
@@ -1155,7 +1157,7 @@ def evaluate_script():
         return _error(result.get("error") if isinstance(result, dict) else "evaluate failed", 500)
     sid = data.get("sid") or next(iter(_SESSIONS), "")
     _queue_event(sid, {"type": "status", "msg": "evaluate", "ts": int(time.time() * 1000)})
-    return _ok(result=result.get("result"), result_type=result.get("result_type"))
+    return _ok(result=result.get("result"), result_type=result.get("result_type"), url=result.get("url"), content_coverage="complete_returned_view")
 
 
 @app.post("/scroll")
@@ -1430,12 +1432,12 @@ def drag():
 def dom_snapshot():
     if not _auth_ok(request):
         return _error("unauthorized", 401)
-    html = Tools.get_dom_snapshot(max_chars=200_000)
+    html = Tools.get_dom_snapshot()
     if not html:
         return _error("no dom (browser closed?)", 409)
     sid = request.args.get("sid") or next(iter(_SESSIONS), "")
     _queue_event(sid, {"type": "dom", "chars": len(html), "ts": int(time.time() * 1000)})
-    return _ok(dom=html, length=len(html))
+    return _ok(dom=html, length=len(html), url=Tools._driver.current_url if Tools._driver else None, content_coverage="complete_returned_view")
 
 
 def _record_frame_meta(sid: str, fname: str, width: int, height: int) -> None:

package/dist/update-worker.js

@@ -250778,7 +250778,7 @@ var MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
 var MAX_SEARCH_RESPONSE_BYTES = 2 * 1024 * 1024;
 
 // packages/execution/dist/tools/web-crawl.js
-var MAX_RESPONSE_BYTES2 = 5 * 1024 * 1024;
+var MAX_CRAWL_RESPONSE_BYTES = 10 * 1024 * 1024;
 
 // packages/execution/dist/tools/osint-catalog-data.js
 var OSINT_CATALOG_RESOURCES = [
@@ -295733,10 +295733,10 @@ for (const resource of OSINT_CATALOG_RESOURCES) {
 }
 
 // packages/execution/dist/tools/carbonyl-browser.js
-init_venv_paths();
-init_process_lifecycle();
 import { homedir as homedir4 } from "node:os";
 import { join as join3 } from "node:path";
+init_venv_paths();
+init_process_lifecycle();
 
 // packages/execution/dist/tools/playwright-browser.js
 import { AsyncLocalStorage as AsyncLocalStorage2 } from "node:async_hooks";
@@ -298686,10 +298686,10 @@ var MAX_IMAGE_BYTES = 8 * 1024 * 1024;
 init_process_async();
 
 // packages/execution/dist/tools/browser-action.js
-init_process_lifecycle();
 import { copyFileSync, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync as readFileSync2 } from "node:fs";
 import { basename as basename2, dirname as dirname4, join as join5, resolve as resolve4 } from "node:path";
 import { fileURLToPath } from "node:url";
+init_process_lifecycle();
 init_model_store();
 var __dirname2 = dirname4(fileURLToPath(import.meta.url));
 var DEFAULT_PORT = 8130;