@agentikos/omega-os 0.19.22 → 0.19.24

package/bootstrap/lib/llm-clis.py

@@ -131,6 +131,25 @@ _CATALOG: list[CliSpec] = [
         install_cmd=["npm", "install", "-g", "@continuedev/cli"],
         description="Continue.dev CLI — multi-provider with config.json",
     ),
+    # v0.19.23 — local LLM runtimes (no API key needed, free local inference).
+    CliSpec(
+        id="ollama", label="Ollama (local model runtime)",
+        bin_name="ollama",
+        install_cmd=[],   # filled at runtime — curl|sh installer
+        description="Local LLM runtime — pull + run Llama/Mistral/Qwen/etc.",
+    ),
+    CliSpec(
+        id="lm_studio", label="LM Studio CLI (local)",
+        bin_name="lms",
+        install_cmd=["npx", "-y", "@lmstudio/cli", "install"],
+        description="LM Studio command-line manager (`lms`) for local models",
+    ),
+    CliSpec(
+        id="gh_copilot", label="GitHub Copilot Chat (via gh extension)",
+        bin_name="gh",   # we extend an existing binary
+        install_cmd=["gh", "extension", "install", "github/gh-copilot"],
+        description="GitHub Copilot Chat as a `gh copilot` subcommand",
+    ),
 ]
 
 

package/bootstrap/lib/steps.sh

@@ -545,6 +545,13 @@ for entry in missing:
     elif "npm_global" in install and shutil.which("npm"):
         rc, out = _run(["npm", "install", "-g", "--silent", install["npm_global"]])
         attempt = f"npm install -g {install['npm_global']}"
+    elif "uv_tool" in install and (shutil.which("uv") or os.path.isfile(
+            os.path.expanduser("~/.local/bin/uv"))):
+        # uv-managed Python tools — installs to ~/.local/share/uv/tools
+        # and links to ~/.local/bin (which step_system_deps put on PATH)
+        uv_bin = shutil.which("uv") or os.path.expanduser("~/.local/bin/uv")
+        rc, out = _run([uv_bin, "tool", "install", "--quiet", install["uv_tool"]])
+        attempt = f"uv tool install {install['uv_tool']}"
     elif "curl" in install:
         # curl|bash installer — last resort
         rc, out = _run(["bash", "-c", f"curl -fsSL {install['curl']} | bash"])
@@ -557,6 +564,15 @@ for entry in missing:
     if shutil.which(binary):
         print(f"  ok   {cli_id:<14} (installed via {attempt})")
         installed_now.append(cli_id)
+        # Optional post-install command (e.g. `scrapling install`
+        # downloads its browser deps after pip install).
+        post = entry.get("post_install") or []
+        if post and shutil.which(post[0]):
+            rc2, _ = _run(post, timeout=600)
+            if rc2 == 0:
+                print(f"       post-install: {' '.join(post)} ok")
+            else:
+                print(f"       post-install: {' '.join(post)} rc={rc2} (non-fatal)")
     else:
         print(f"  fail {cli_id}: {attempt} returned rc={rc}")
         failed_now.append(cli_id)

package/omega/Agentik_Engine/omega_engine/__init__.py

@@ -188,7 +188,7 @@ from omega_engine.genesis import (
 )
 from omega_engine import plan as plan_v7
 
-__version__ = "0.19.22"
+__version__ = "0.19.24"
 
 __all__ = [
     "__version__",

package/omega/Agentik_Engine/omega_engine/cli.py

@@ -2965,6 +2965,132 @@ def _set_active_provider(provider_id: str) -> None:
     p.write_text(provider_id.strip() + "\n")
 
 
+def cmd_scrape(args: argparse.Namespace) -> int:
+    """`omega scrape <url> [--engine cloak|scrapling] [--out file]` — scraper.
+
+    Two engines available since v0.19.24:
+      * ``cloak`` (default) — CloakBrowser, heavy stealth Chromium, 58
+        C++ patches, passes Cloudflare/Turnstile/FingerprintJS. Slow
+        but invincible.
+      * ``scrapling`` — D4Vinci/Scrapling, HTTP-first with optional
+        browser, adaptive element tracking, concurrent crawl with
+        pause/resume. Fast for non-protected sites.
+
+    Output: cleaned text to stdout (or --out FILE). Wraps both engines
+    behind a consistent CLI so any LLM (Claude/Gemini/Codex/OpenCode/
+    Aider) can `omega scrape <url>` via Bash without caring which
+    Python library is doing the work.
+
+    Examples:
+      omega scrape https://example.com                       # CloakBrowser
+      omega scrape https://example.com --engine scrapling    # Scrapling
+      omega scrape https://news.ycombinator.com --engine scrapling --css ".titleline"
+    """
+    import shutil
+    import subprocess
+    if not args.url:
+        print("usage: omega scrape <url> "
+              "[--engine cloak|scrapling] [--out FILE] [--humanize] "
+              "[--proxy URL] [--css SELECTOR]")
+        return 2
+    engine = getattr(args, "engine", None) or "cloak"
+    if engine == "scrapling":
+        # Scrapling path — uses its own CLI `scrapling extract`.
+        if not shutil.which("scrapling"):
+            print("  scrapling not installed — "
+                  "run `omega tool install scrapling` or "
+                  "`uv tool install 'scrapling[fetchers]' && scrapling install`")
+            return 2
+        out_file = getattr(args, "out", None)
+        # Scrapling writes to a file (it doesn't print to stdout by
+        # default). If user didn't specify --out, use a tmpfile and
+        # cat it after.
+        import tempfile
+        from pathlib import Path
+        tmp_out = out_file or tempfile.mktemp(suffix=".md")
+        scrapling_cmd = ["scrapling", "extract",
+                         "stealthy-fetch" if getattr(args, "humanize", False)
+                         else "fetch",
+                         args.url, tmp_out]
+        if getattr(args, "css", None):
+            scrapling_cmd += ["--css-selector", args.css]
+        try:
+            proc = subprocess.run(scrapling_cmd, capture_output=True,
+                                  text=True, timeout=120)
+        except subprocess.TimeoutExpired:
+            print("  scrapling timed out after 120s")
+            return 2
+        if proc.returncode != 0:
+            print(f"  scrapling error: {proc.stderr[:400]}")
+            return proc.returncode
+        if not out_file:
+            try:
+                print(Path(tmp_out).read_text())
+                Path(tmp_out).unlink(missing_ok=True)
+            except OSError as exc:
+                print(f"  read scrapling output failed: {exc}")
+                return 2
+        else:
+            print(f"  wrote → {out_file}")
+        return 0
+    # Default engine = CloakBrowser (kept verbatim from v0.19.23).
+    # CloakBrowser is installed as a uv tool (`cloakbrowser`) but we need
+    # to drive it from Python — the canonical path is `python -m
+    # cloakbrowser` after `pip install cloakbrowser`, OR we invoke the
+    # uv-tool-installed Python directly. Try shutil.which("cloakbrowser")
+    # first; if absent, fall back to running through the installer venv.
+    runner = None
+    if shutil.which("cloakbrowser"):
+        # Direct CLI is mostly for binary management (install/info/update).
+        # For SCRAPING we still need our Python wrapper.
+        pass
+    # Inline Python wrapper — no separate script file, no version drift.
+    py_code = (
+        "import sys, asyncio\n"
+        "try:\n"
+        "    from cloakbrowser import launch_async\n"
+        "except ImportError:\n"
+        "    sys.stderr.write('cloakbrowser not importable — install with: "
+        "uv tool install cloakbrowser\\n'); sys.exit(2)\n"
+        "URL = sys.argv[1]\n"
+        "HUMANIZE = '--humanize' in sys.argv\n"
+        "PROXY = None\n"
+        "if '--proxy' in sys.argv:\n"
+        "    PROXY = sys.argv[sys.argv.index('--proxy') + 1]\n"
+        "async def main():\n"
+        "    kwargs = {'humanize': HUMANIZE}\n"
+        "    if PROXY:\n"
+        "        kwargs['proxy'] = PROXY\n"
+        "    browser = await launch_async(**kwargs)\n"
+        "    page = await browser.new_page()\n"
+        "    await page.goto(URL, wait_until='networkidle')\n"
+        "    text = await page.evaluate(\"document.body.innerText\")\n"
+        "    await browser.close()\n"
+        "    print(text)\n"
+        "asyncio.run(main())\n"
+    )
+    cmd = ["python3", "-c", py_code, args.url]
+    if getattr(args, "humanize", False):
+        cmd.append("--humanize")
+    if getattr(args, "proxy", None):
+        cmd += ["--proxy", args.proxy]
+    try:
+        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
+    except subprocess.TimeoutExpired:
+        print("  cloakbrowser timed out after 120s")
+        return 2
+    out_target = getattr(args, "out", None)
+    if out_target:
+        from pathlib import Path
+        Path(out_target).write_text(proc.stdout)
+        print(f"  wrote {len(proc.stdout)} bytes → {out_target}")
+    else:
+        print(proc.stdout)
+    if proc.returncode != 0:
+        print(f"  cloakbrowser stderr: {proc.stderr[:400]}", file=__import__('sys').stderr)
+    return proc.returncode
+
+
 def cmd_switch(args: argparse.Namespace) -> int:
     """`omega switch <provider>` — hot-swap the LLM provider for new chats.
 
@@ -4109,6 +4235,30 @@ def _build_parser() -> argparse.ArgumentParser:
     p_sw.add_argument("provider", nargs="?", default=None,
                       help="provider id (omit to print current + available)")
     p_sw.set_defaults(fn=cmd_switch)
+
+    # `omega scrape <url>` — official OmegaOS web scraper (CloakBrowser
+    # default, Scrapling optional). Both engines pre-installed at step 40
+    # when their catalog entry is `recommended: true`.
+    p_sc = sub.add_parser(
+        "scrape",
+        help="scrape a URL — CloakBrowser (default, stealth) or Scrapling "
+        "(--engine scrapling, fast). Output: stdout or --out FILE",
+    )
+    p_sc.add_argument("url", nargs="?", default=None,
+                      help="URL to fetch")
+    p_sc.add_argument("--engine", default="cloak",
+                      choices=["cloak", "scrapling"],
+                      help="which scraper to use (default: cloak = CloakBrowser)")
+    p_sc.add_argument("--out", default=None,
+                      help="write result to file instead of stdout")
+    p_sc.add_argument("--humanize", action="store_true",
+                      help="mouse curves + scroll patterns (cloak) "
+                      "or stealthy-fetch mode (scrapling)")
+    p_sc.add_argument("--proxy", default=None,
+                      help="HTTP or SOCKS5 proxy (cloak engine only)")
+    p_sc.add_argument("--css", default=None,
+                      help="CSS selector to scope output (scrapling engine only)")
+    p_sc.set_defaults(fn=cmd_scrape)
     p_doc = sub.add_parser("doctor", help="validate the deployment")
     p_doc.add_argument("--json", action="store_true",
                        help="emit a single JSON object instead of pretty text")

package/omega/Agentik_Engine/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "omega-engine"
-version = "0.19.22"
+version = "0.19.24"
 description = "The Omega OS orchestration engine — event-sourced, verified-completion agent graphs."
 readme = "README.md"
 requires-python = ">=3.11"

package/omega/Agentik_SSOT/VERSION

@@ -1 +1 @@
-0.19.22
+0.19.24

package/omega/Agentik_SSOT/clis/clis-catalog.yaml

@@ -64,6 +64,29 @@ native:
     secrets: []
     recommended: true
 
+  - id: cloakbrowser
+    name: CloakBrowser (official OmegaOS scraper — bypasses bot detection)
+    # PyPI package — installed into a uv tool so it's globally available
+    # as `cloakbrowser` + `python -m cloakbrowser`. ~200MB binary
+    # downloaded on first run (cached).
+    install: { uv_tool: "cloakbrowser" }
+    binary: cloakbrowser
+    secrets: []
+    recommended: true
+
+  - id: scrapling
+    name: Scrapling (optional fast scraper — HTTP-first, adaptive elements)
+    # Complementary to CloakBrowser. Scrapling shines for non-protected
+    # sites + multi-page crawls (concurrent + pause/resume + adaptive
+    # element tracking that survives page redesigns). CloakBrowser stays
+    # the default for hard bot-walls; Scrapling for everything else.
+    # `omega scrape --engine scrapling` selects it explicitly.
+    install: { uv_tool: "scrapling[fetchers]" }
+    binary: scrapling
+    secrets: []
+    recommended: false
+    post_install: ["scrapling", "install"]   # downloads browser deps once
+
 # --- 2. Printing Press CLIs --------------------------------------------
 # Installed via `npx -y @mvanhorn/printing-press-library install <name>`.
 # Each ships with a local SQLite mirror + Claude Code skill.

package/omega/Agentik_SSOT/llm-providers/providers-catalog.yaml

@@ -0,0 +1,169 @@
+# Omega OS — LLM provider catalog (v0.19.23+)
+#
+# Maps the operator's intent ("I want to talk to GLM") to a concrete
+# (cli + provider_config) pair. Most providers don't ship their own CLI —
+# they're configurable backends inside OpenCode / Continue.dev / Aider.
+#
+# Used by:
+#   * `omega switch <provider>` — looks up the cli + writes the
+#     provider-active marker.
+#   * step_provider_configs — drops templated config files for every
+#     supported provider with secret refs from the vault.
+#   * `omega account login --provider <id>` — guides the auth flow per
+#     provider.
+#
+# Secrets are vault refs, never literal. Templated config files end up at
+# the LLM CLI's expected path on first launch.
+
+version: 1
+
+providers:
+  # -- Anthropic family --
+  - id: anthropic
+    label: "Anthropic Claude (API key)"
+    cli: claude_code
+    auth_kind: api_key
+    secret_refs: [ANTHROPIC_API_KEY]
+    notes: |
+      Direct Anthropic API access. Pay-per-token. For OmegaOS's own
+      L3-L5 stack we prefer Claude Max OAuth (no cost) — this entry is
+      for L2 / Hermès style "use my paid Anthropic budget" scenarios.
+
+  - id: chatgpt_subscription
+    label: "ChatGPT Subscription (OpenAI Codex)"
+    cli: codex
+    auth_kind: oauth
+    secret_refs: []
+    notes: |
+      Codex CLI uses the same OAuth as your ChatGPT Plus/Team sub —
+      no separate API key needed. `codex login` opens the browser.
+
+  # -- OpenAI family --
+  - id: openai
+    label: "OpenAI (API key — gpt-4o, gpt-5, o3-pro)"
+    cli: codex          # codex IS OpenAI's official agentic CLI
+    auth_kind: api_key
+    secret_refs: [OPENAI_API_KEY]
+    notes: |
+      Pay-per-token OpenAI API. Use this when you want gpt-5/o3-pro
+      without the ChatGPT subscription cap.
+
+  - id: openai_compatible
+    label: "OpenAI API-Compatible endpoint (custom)"
+    cli: opencode       # OpenCode's `provider: openai_compatible`
+    auth_kind: api_key
+    secret_refs: [OPENAI_API_BASE, OPENAI_API_KEY]
+    notes: |
+      For LM Studio, vLLM, llama.cpp's `/v1` server, Together.ai's
+      OpenAI-shape endpoint, etc.
+
+  # -- Google --
+  - id: google_ai
+    label: "Google AI Studio (Gemini)"
+    cli: gemini_cli
+    auth_kind: api_key
+    secret_refs: [GOOGLE_API_KEY]
+    notes: |
+      Direct Gemini API via the official `gemini` CLI. Free tier
+      generous; paid tier for higher RPM.
+
+  # -- China --
+  - id: glm
+    label: "Zhipu GLM-4.5 / GLM-4.6 (via OpenCode)"
+    cli: opencode
+    auth_kind: api_key
+    secret_refs: [GLM_API_KEY]
+    notes: |
+      Zhipu AI's GLM models — open weights for the smaller variants,
+      hosted API for GLM-4.6. OpenCode provider `zhipu`.
+
+  - id: deepseek
+    label: "DeepSeek V3 / R1 (via OpenCode)"
+    cli: opencode
+    auth_kind: api_key
+    secret_refs: [DEEPSEEK_API_KEY]
+    notes: |
+      DeepSeek's V3 + R1 reasoning. Excellent price/perf. OpenCode
+      provider `deepseek`.
+
+  - id: qwen
+    label: "Qwen (Alibaba) — dedicated CLI"
+    cli: qwen_code
+    auth_kind: api_key
+    secret_refs: [DASHSCOPE_API_KEY]
+    notes: |
+      Qwen3-Coder via Alibaba's DashScope. Standalone `qwen` CLI.
+
+  # -- Local --
+  - id: ollama
+    label: "Ollama (local — Llama/Mistral/Qwen/DeepSeek-Coder)"
+    cli: ollama
+    auth_kind: none
+    secret_refs: []
+    notes: |
+      Local model runtime. Free, private, runs offline. Pull a model
+      with `ollama pull llama3.3:70b` then `ollama run`.
+
+  - id: lm_studio
+    label: "LM Studio (local — GUI + CLI)"
+    cli: lm_studio
+    auth_kind: none
+    secret_refs: []
+    notes: |
+      LM Studio for desktop. Run any GGUF model locally. Exposes an
+      OpenAI-compatible `/v1` endpoint OpenCode can consume.
+
+  # -- Aggregators / Gateways --
+  - id: openrouter
+    label: "OpenRouter (300+ models, one API key)"
+    cli: opencode
+    auth_kind: api_key
+    secret_refs: [OPENROUTER_API_KEY]
+    notes: |
+      Route to any of 300+ models with one API key. OpenCode provider
+      `openrouter`. Per-call routing in config.
+
+  - id: vercel_ai_gateway
+    label: "Vercel AI Gateway (multi-provider)"
+    cli: opencode
+    auth_kind: api_key
+    secret_refs: [AI_GATEWAY_API_KEY]
+    notes: |
+      Vercel's hosted gateway proxies your traffic across providers
+      with usage caps + observability. OpenCode `provider: ai_gateway`.
+
+  # -- Hyperscalers --
+  - id: amazon_bedrock
+    label: "Amazon Bedrock (AWS)"
+    cli: opencode
+    auth_kind: aws
+    secret_refs: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION]
+    notes: |
+      AWS Bedrock — Claude, Llama, Mistral, Cohere, Stability through
+      AWS auth. OpenCode `provider: bedrock`.
+
+  # -- Other --
+  - id: mistral
+    label: "Mistral AI (API key)"
+    cli: opencode
+    auth_kind: api_key
+    secret_refs: [MISTRAL_API_KEY]
+    notes: |
+      Mistral-Large, Codestral, Pixtral. OpenCode `provider: mistral`.
+
+  - id: xai
+    label: "xAI Grok (Grok-2, Grok-Beta)"
+    cli: opencode
+    auth_kind: api_key
+    secret_refs: [XAI_API_KEY]
+    notes: |
+      Grok-2 / Grok-Beta via xAI's API. OpenCode `provider: xai`.
+
+  - id: github_copilot
+    label: "GitHub Copilot Chat (via gh extension)"
+    cli: gh_copilot
+    auth_kind: oauth
+    secret_refs: []
+    notes: |
+      Talk to Copilot from the terminal — `gh copilot suggest "..."` and
+      `gh copilot explain "..."`. Uses your GitHub OAuth.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentikos/omega-os",
-  "version": "0.19.22",
+  "version": "0.19.24",
   "description": "Omega OS — installable agentic operating system with verified-completion orchestration. Event-sourced engine, 8-block rack, autonomous agents, MCP.",
   "bin": {
     "omega-os": "bin/omega-os.js"