@agentikos/omega-os 0.19.23 → 0.19.24

package/bootstrap/lib/steps.sh

@@ -564,6 +564,15 @@ for entry in missing:
     if shutil.which(binary):
         print(f"  ok   {cli_id:<14} (installed via {attempt})")
         installed_now.append(cli_id)
+        # Optional post-install command (e.g. `scrapling install`
+        # downloads its browser deps after pip install).
+        post = entry.get("post_install") or []
+        if post and shutil.which(post[0]):
+            rc2, _ = _run(post, timeout=600)
+            if rc2 == 0:
+                print(f"       post-install: {' '.join(post)} ok")
+            else:
+                print(f"       post-install: {' '.join(post)} rc={rc2} (non-fatal)")
     else:
         print(f"  fail {cli_id}: {attempt} returned rc={rc}")
         failed_now.append(cli_id)

package/omega/Agentik_Engine/omega_engine/__init__.py

@@ -188,7 +188,7 @@ from omega_engine.genesis import (
 )
 from omega_engine import plan as plan_v7
 
-__version__ = "0.19.23"
+__version__ = "0.19.24"
 
 __all__ = [
     "__version__",

package/omega/Agentik_Engine/omega_engine/cli.py

@@ -2966,28 +2966,74 @@ def _set_active_provider(provider_id: str) -> None:
 
 
 def cmd_scrape(args: argparse.Namespace) -> int:
-    """`omega scrape <url> [--out file] [--humanize]` — stealth scraper.
-
-    Thin wrapper over CloakBrowser (https://github.com/CloakHQ/CloakBrowser).
-    Bypasses Cloudflare/Turnstile/FingerprintJS by patching the browser at
-    the C++ level. Default output = markdown to stdout so any LLM CLI can
-    pipe it directly into a context. The wrapper is what makes
-    CloakBrowser usable as a SHELL CLI for Claude/Gemini/Codex/etc.
-
-    Usage:
-      omega scrape https://example.com                    # → markdown to stdout
-      omega scrape https://example.com --out page.md      # → file
-      omega scrape https://example.com --humanize         # mouse curves + scroll
-      omega scrape https://example.com --proxy <url>      # SOCKS5/HTTP
-
-    Inside any Claude Code chat session: ``omega scrape <url>`` returns the
-    cleaned page content. No MCP server, no Playwright dance.
+    """`omega scrape <url> [--engine cloak|scrapling] [--out file]` — scraper.
+
+    Two engines available since v0.19.24:
+      * ``cloak`` (default) — CloakBrowser, heavy stealth Chromium, 58
+        C++ patches, passes Cloudflare/Turnstile/FingerprintJS. Slow
+        but invincible.
+      * ``scrapling`` — D4Vinci/Scrapling, HTTP-first with optional
+        browser, adaptive element tracking, concurrent crawl with
+        pause/resume. Fast for non-protected sites.
+
+    Output: cleaned text to stdout (or --out FILE). Wraps both engines
+    behind a consistent CLI so any LLM (Claude/Gemini/Codex/OpenCode/
+    Aider) can `omega scrape <url>` via Bash without caring which
+    Python library is doing the work.
+
+    Examples:
+      omega scrape https://example.com                       # CloakBrowser
+      omega scrape https://example.com --engine scrapling    # Scrapling
+      omega scrape https://news.ycombinator.com --engine scrapling --css ".titleline"
     """
     import shutil
     import subprocess
     if not args.url:
-        print("usage: omega scrape <url> [--out FILE] [--humanize] [--proxy URL]")
+        print("usage: omega scrape <url> "
+              "[--engine cloak|scrapling] [--out FILE] [--humanize] "
+              "[--proxy URL] [--css SELECTOR]")
         return 2
+    engine = getattr(args, "engine", None) or "cloak"
+    if engine == "scrapling":
+        # Scrapling path — uses its own CLI `scrapling extract`.
+        if not shutil.which("scrapling"):
+            print("  scrapling not installed — "
+                  "run `omega tool install scrapling` or "
+                  "`uv tool install 'scrapling[fetchers]' && scrapling install`")
+            return 2
+        out_file = getattr(args, "out", None)
+        # Scrapling writes to a file (it doesn't print to stdout by
+        # default). If user didn't specify --out, use a tmpfile and
+        # cat it after.
+        import tempfile
+        from pathlib import Path
+        tmp_out = out_file or tempfile.mktemp(suffix=".md")
+        scrapling_cmd = ["scrapling", "extract",
+                         "stealthy-fetch" if getattr(args, "humanize", False)
+                         else "fetch",
+                         args.url, tmp_out]
+        if getattr(args, "css", None):
+            scrapling_cmd += ["--css-selector", args.css]
+        try:
+            proc = subprocess.run(scrapling_cmd, capture_output=True,
+                                  text=True, timeout=120)
+        except subprocess.TimeoutExpired:
+            print("  scrapling timed out after 120s")
+            return 2
+        if proc.returncode != 0:
+            print(f"  scrapling error: {proc.stderr[:400]}")
+            return proc.returncode
+        if not out_file:
+            try:
+                print(Path(tmp_out).read_text())
+                Path(tmp_out).unlink(missing_ok=True)
+            except OSError as exc:
+                print(f"  read scrapling output failed: {exc}")
+                return 2
+        else:
+            print(f"  wrote → {out_file}")
+        return 0
+    # Default engine = CloakBrowser (kept verbatim from v0.19.23).
     # CloakBrowser is installed as a uv tool (`cloakbrowser`) but we need
     # to drive it from Python — the canonical path is `python -m
     # cloakbrowser` after `pip install cloakbrowser`, OR we invoke the
@@ -4190,20 +4236,28 @@ def _build_parser() -> argparse.ArgumentParser:
                       help="provider id (omit to print current + available)")
     p_sw.set_defaults(fn=cmd_switch)
 
-    # `omega scrape <url>` — official OmegaOS web scraper (CloakBrowser).
+    # `omega scrape <url>` — official OmegaOS web scraper (CloakBrowser
+    # default, Scrapling optional). Both engines pre-installed at step 40
+    # when their catalog entry is `recommended: true`.
     p_sc = sub.add_parser(
         "scrape",
-        help="stealth scrape a URL via CloakBrowser — bypasses Cloudflare, "
-        "Turnstile, FingerprintJS. Output: markdown to stdout (or --out FILE)",
+        help="scrape a URL — CloakBrowser (default, stealth) or Scrapling "
+        "(--engine scrapling, fast). Output: stdout or --out FILE",
     )
     p_sc.add_argument("url", nargs="?", default=None,
                       help="URL to fetch")
+    p_sc.add_argument("--engine", default="cloak",
+                      choices=["cloak", "scrapling"],
+                      help="which scraper to use (default: cloak = CloakBrowser)")
     p_sc.add_argument("--out", default=None,
                       help="write result to file instead of stdout")
     p_sc.add_argument("--humanize", action="store_true",
-                      help="mouse curves + keyboard timing + scroll patterns")
+                      help="mouse curves + scroll patterns (cloak) "
+                      "or stealthy-fetch mode (scrapling)")
     p_sc.add_argument("--proxy", default=None,
-                      help="HTTP or SOCKS5 proxy (with inline creds)")
+                      help="HTTP or SOCKS5 proxy (cloak engine only)")
+    p_sc.add_argument("--css", default=None,
+                      help="CSS selector to scope output (scrapling engine only)")
     p_sc.set_defaults(fn=cmd_scrape)
     p_doc = sub.add_parser("doctor", help="validate the deployment")
     p_doc.add_argument("--json", action="store_true",

package/omega/Agentik_Engine/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "omega-engine"
-version = "0.19.23"
+version = "0.19.24"
 description = "The Omega OS orchestration engine — event-sourced, verified-completion agent graphs."
 readme = "README.md"
 requires-python = ">=3.11"

package/omega/Agentik_SSOT/VERSION

@@ -1 +1 @@
-0.19.23
+0.19.24

package/omega/Agentik_SSOT/clis/clis-catalog.yaml

@@ -74,6 +74,19 @@ native:
     secrets: []
     recommended: true
 
+  - id: scrapling
+    name: Scrapling (optional fast scraper — HTTP-first, adaptive elements)
+    # Complementary to CloakBrowser. Scrapling shines for non-protected
+    # sites + multi-page crawls (concurrent + pause/resume + adaptive
+    # element tracking that survives page redesigns). CloakBrowser stays
+    # the default for hard bot-walls; Scrapling for everything else.
+    # `omega scrape --engine scrapling` selects it explicitly.
+    install: { uv_tool: "scrapling[fetchers]" }
+    binary: scrapling
+    secrets: []
+    recommended: false
+    post_install: ["scrapling", "install"]   # downloads browser deps once
+
 # --- 2. Printing Press CLIs --------------------------------------------
 # Installed via `npx -y @mvanhorn/printing-press-library install <name>`.
 # Each ships with a local SQLite mirror + Claude Code skill.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentikos/omega-os",
-  "version": "0.19.23",
+  "version": "0.19.24",
   "description": "Omega OS — installable agentic operating system with verified-completion orchestration. Event-sourced engine, 8-block rack, autonomous agents, MCP.",
   "bin": {
     "omega-os": "bin/omega-os.js"