@oneciel-ai/ciel-runtime 0.1.1 → 0.2.0

package/ciel_runtime_support/visible_stream_filters.py

@@ -0,0 +1,130 @@
+"""Stateful removal of provider-visible thinking and tool-call artifacts."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+VISIBLE_THINKING_MARKUP_TAG_RE = re.compile(r"</?think(?:ing)?\b[^>]*>", re.I)
+VISIBLE_THINKING_MARKUP_PREFIXES = ("<think", "</think", "<thinking", "</thinking")
+VISIBLE_TOOL_CALL_ARTIFACT_SUFFIX_RE = re.compile(
+    r"(?s)(?:^|(?:\r?\n){1,3})[ \t]*call[ \t]*(?:\r?\n)"
+    r"[ \t]*ignore[ \t]*(?:\r?\n)?[ \t]*$"
+)
+VISIBLE_TOOL_CALL_ARTIFACT_HOLD_CHARS = 96
+
+
+def visible_thinking_markup_partial_start(text: str) -> int:
+    less_than = text.rfind("<")
+    if less_than < 0 or less_than <= text.rfind(">"):
+        return -1
+    suffix = text[less_than:].lower()
+    if any(
+        prefix.startswith(suffix) or suffix.startswith(prefix)
+        for prefix in VISIBLE_THINKING_MARKUP_PREFIXES
+    ):
+        return less_than
+    return -1
+
+
+class VisibleThinkingMarkupFilter:
+    def __init__(self) -> None:
+        self.in_thinking = False
+        self.pending = ""
+
+    def feed(self, text: Any) -> str:
+        raw = str(text or "")
+        if not raw:
+            return ""
+        value = self.pending + raw
+        self.pending = ""
+        partial_start = visible_thinking_markup_partial_start(value)
+        if partial_start >= 0:
+            self.pending = value[partial_start:]
+            value = value[:partial_start]
+        return self._strip_complete_tags(value)
+
+    def finish(self) -> str:
+        pending = self.pending
+        self.pending = ""
+        if (
+            not pending
+            or self.in_thinking
+            or visible_thinking_markup_partial_start(pending) == 0
+        ):
+            self.in_thinking = False
+            return ""
+        return self._strip_complete_tags(pending)
+
+    def _strip_complete_tags(self, text: str) -> str:
+        output: list[str] = []
+        position = 0
+        for match in VISIBLE_THINKING_MARKUP_TAG_RE.finditer(text):
+            closing = match.group(0).lstrip().lower().startswith("</")
+            if self.in_thinking:
+                if closing:
+                    self.in_thinking = False
+                position = match.end()
+                continue
+            output.append(text[position : match.start()])
+            position = match.end()
+            if not closing:
+                self.in_thinking = True
+        if not self.in_thinking:
+            output.append(text[position:])
+        return "".join(output)
+
+
+def strip_visible_thinking_markup(text: Any) -> str:
+    raw = str(text or "")
+    if "<think" not in raw.lower() and "</think" not in raw.lower():
+        return raw
+    filter_state = VisibleThinkingMarkupFilter()
+    return (filter_state.feed(raw) + filter_state.finish()).lstrip()
+
+
+def strip_visible_tool_call_artifact_suffix(text: Any) -> str:
+    raw = str(text or "")
+    if "call" not in raw or "ignore" not in raw:
+        return raw
+    return VISIBLE_TOOL_CALL_ARTIFACT_SUFFIX_RE.sub("", raw)
+
+
+class VisibleToolCallArtifactFilter:
+    def __init__(self, hold_chars: int = VISIBLE_TOOL_CALL_ARTIFACT_HOLD_CHARS) -> None:
+        self.hold_chars = max(16, int(hold_chars))
+        self.pending = ""
+        self.stripped = False
+
+    def feed(self, text: Any) -> str:
+        raw = str(text or "")
+        if not raw:
+            return ""
+        value = self.pending + raw
+        if len(value) <= self.hold_chars:
+            self.pending = value
+            return ""
+        emit_length = len(value) - self.hold_chars
+        self.pending = value[emit_length:]
+        return value[:emit_length]
+
+    def finish(self) -> str:
+        pending = self.pending
+        self.pending = ""
+        stripped = strip_visible_tool_call_artifact_suffix(pending)
+        self.stripped = self.stripped or stripped != pending
+        return stripped
+
+
+__all__ = [
+    "VISIBLE_THINKING_MARKUP_PREFIXES",
+    "VISIBLE_THINKING_MARKUP_TAG_RE",
+    "VISIBLE_TOOL_CALL_ARTIFACT_HOLD_CHARS",
+    "VISIBLE_TOOL_CALL_ARTIFACT_SUFFIX_RE",
+    "VisibleThinkingMarkupFilter",
+    "VisibleToolCallArtifactFilter",
+    "strip_visible_thinking_markup",
+    "strip_visible_tool_call_artifact_suffix",
+    "visible_thinking_markup_partial_start",
+]

package/ciel_runtime_support/web_endpoints.py

@@ -0,0 +1,447 @@
+"""Startup web endpoint options and optional Tailscale Serve integration."""
+
+from __future__ import annotations
+
+import ipaddress
+import json
+import os
+import shutil
+import subprocess
+import urllib.parse
+from collections.abc import MutableMapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+_TRUE = {"1", "true", "yes", "on"}
+_WILDCARD_HOSTS = {"0.0.0.0", "::", "[::]"}
+
+
+@dataclass(frozen=True, slots=True)
+class TailscaleNode:
+    ipv4: str
+    dns_name: str
+    online: bool = True
+
+
+@dataclass(frozen=True, slots=True)
+class WebEndpointReport:
+    local_url: str
+    tailscale_ip_url: str = ""
+    tailscale_https_url: str = ""
+    tailscale_https_command: str = ""
+
+    def status_lines(self) -> list[str]:
+        lines = [f"web: {self.local_url}"]
+        if self.tailscale_ip_url:
+            lines.append(f"web_tailscale_http: {self.tailscale_ip_url}")
+        if self.tailscale_https_url:
+            lines.append(f"web_tailscale_https: {self.tailscale_https_url}")
+        elif self.tailscale_https_command:
+            lines.append(f"web_tailscale_https_recommended: {self.tailscale_https_command}")
+        return lines
+
+
+@dataclass(frozen=True, slots=True)
+class WebBackendSettings:
+    enabled: bool = False
+    host: str = "127.0.0.1"
+    port: int = 0
+    tailscale_https: bool = False
+
+    @property
+    def client_host(self) -> str:
+        return "127.0.0.1" if self.host in _WILDCARD_HOSTS else self.host
+
+
+def web_backend_settings(config: dict[str, Any] | None) -> WebBackendSettings:
+    raw = config.get("web_backend") if isinstance(config, dict) else {}
+    values = raw if isinstance(raw, dict) else {}
+    host = str(values.get("host") or "127.0.0.1").strip()
+    if not host or any(character.isspace() for character in host):
+        host = "127.0.0.1"
+    try:
+        port = int(values.get("port") or 0)
+    except (TypeError, ValueError):
+        port = 0
+    if not 1 <= port <= 65535:
+        port = 0
+    tailscale_https = bool(values.get("tailscale_https", False))
+    enabled_value = values.get("enabled")
+    enabled = (
+        bool(enabled_value)
+        if enabled_value is not None
+        else bool(tailscale_https or host != "127.0.0.1" or port)
+    )
+    return WebBackendSettings(enabled, host, port, tailscale_https)
+
+
+def load_saved_web_backend(path: os.PathLike[str] | str) -> WebBackendSettings:
+    try:
+        payload = json.loads(Path(path).read_text(encoding="utf-8"))
+    except (OSError, ValueError, TypeError):
+        payload = {}
+    return web_backend_settings(payload if isinstance(payload, dict) else {})
+
+
+def web_backend_summary(config: dict[str, Any], effective_port: int) -> str:
+    settings = web_backend_settings(config)
+    port = settings.port or effective_port or int(config.get("_effective_web_port") or 0)
+    mode = " + Tailscale HTTPS" if settings.tailscale_https else ""
+    state = "on" if settings.enabled else "off"
+    return f"{state} · {settings.host}:{port or 'auto'}{mode}"
+
+
+def web_backend_panel_rows(
+    config: dict[str, Any], effective_port: int
+) -> tuple[list[str], list[str]]:
+    settings = web_backend_settings(config)
+    port = settings.port or effective_port
+    node = discover_tailscale_node()
+    tailscale = "on" if settings.tailscale_https else "off"
+    if node and node.dns_name:
+        tailscale += f" · {node.dns_name}:{port}"
+    else:
+        tailscale += " · not detected"
+    return (
+        [
+            f"Web backend  [{'on' if settings.enabled else 'off'}]",
+            f"Web bind address  [{settings.host}]",
+            f"Web port  [{port}]",
+            f"Tailscale HTTPS  [{tailscale}]",
+            "Back",
+        ],
+        ["enabled", "host", "port", "tailscale", "back"],
+    )
+
+
+def update_web_backend_config(
+    config: dict[str, Any], key: str, value: Any, effective_port: int
+) -> list[str]:
+    current = web_backend_settings(config)
+    enabled = current.enabled
+    host = current.host
+    port = current.port
+    tailscale_https = current.tailscale_https
+    if key == "enabled":
+        enabled = bool(value)
+    elif key == "host":
+        host, embedded_port = _normalize_web_address(str(value))
+        enabled = True
+        if embedded_port is not None:
+            port = embedded_port
+    elif key == "port":
+        port = _valid_port(str(value), "web port")
+        enabled = True
+    elif key == "tailscale":
+        tailscale_https = bool(value)
+        enabled = True
+    else:
+        raise ValueError(f"unknown web backend setting: {key}")
+    config["web_backend"] = {
+        "enabled": enabled,
+        "host": host,
+        "port": port or effective_port,
+        "tailscale_https": tailscale_https,
+    }
+    external = host in _WILDCARD_HOSTS or not _is_loopback(host)
+    config["router_debug_external_access"] = external
+    config["router_debug_external_access_confirmed"] = external
+    scheme = "https" if tailscale_https else "http"
+    return [
+        f"Web backend: {'on' if enabled else 'off'} · {host}:{port or effective_port}.",
+        f"Tailscale HTTPS: {'on' if tailscale_https else 'off'}.",
+        f"Runtime is restarting so {scheme} endpoint settings apply now.",
+    ]
+
+
+def apply_startup_web_options(
+    argv: list[str],
+    environ: MutableMapping[str, str],
+) -> list[str]:
+    """Consume Ciel-owned web flags before runtime path constants are imported."""
+    output = [argv[0]] if argv else []
+    index = 1
+    passthrough = False
+    while index < len(argv):
+        argument = argv[index]
+        if passthrough:
+            output.append(argument)
+            index += 1
+            continue
+        if argument == "--":
+            passthrough = True
+            output.append(argument)
+            index += 1
+            continue
+        name, separator, inline_value = argument.partition("=")
+        if name in {"--ca-web-address", "--ca-web-host"}:
+            value, index = _option_value(argv, index, inline_value if separator else "", name)
+            host, embedded_port = _normalize_web_address(value)
+            _apply_web_host(host, environ)
+            environ["CIEL_RUNTIME_WEB_START_REQUESTED"] = "1"
+            if embedded_port is not None:
+                environ["CIEL_RUNTIME_ROUTER_PORT"] = str(embedded_port)
+            continue
+        if name == "--ca-web-port":
+            value, index = _option_value(argv, index, inline_value if separator else "", name)
+            environ["CIEL_RUNTIME_ROUTER_PORT"] = str(_valid_port(value, name))
+            environ["CIEL_RUNTIME_WEB_START_REQUESTED"] = "1"
+            continue
+        if name == "--ca-tailscale-https":
+            environ["CIEL_RUNTIME_TAILSCALE_HTTPS"] = "1"
+            environ["CIEL_RUNTIME_WEB_START_REQUESTED"] = "1"
+            if separator and inline_value:
+                environ["CIEL_RUNTIME_TAILSCALE_HTTPS_PORT"] = str(
+                    _valid_port(inline_value, name)
+                )
+            index += 1
+            continue
+        output.append(argument)
+        index += 1
+    return output
+
+
+def _option_value(
+    argv: Sequence[str], index: int, inline_value: str, option: str
+) -> tuple[str, int]:
+    if inline_value:
+        return inline_value, index + 1
+    if index + 1 >= len(argv) or argv[index + 1] == "--":
+        raise SystemExit(f"{option} requires a value")
+    return argv[index + 1], index + 2
+
+
+def _normalize_web_address(value: str) -> tuple[str, int | None]:
+    text = value.strip()
+    if not text:
+        raise SystemExit("--ca-web-address requires a host or URL")
+    parsed = urllib.parse.urlparse(text if "://" in text else f"//{text}")
+    if parsed.scheme and parsed.scheme not in {"http", "https"}:
+        raise SystemExit("--ca-web-address supports only http:// or https:// URLs")
+    if parsed.path not in {"", "/"} or parsed.query or parsed.fragment:
+        raise SystemExit("--ca-web-address must not include a path, query, or fragment")
+    try:
+        port = parsed.port
+    except ValueError as exc:
+        raise SystemExit(f"invalid web address port: {exc}") from exc
+    host = str(parsed.hostname or "").strip()
+    if not host or any(character.isspace() for character in host):
+        raise SystemExit("--ca-web-address contains an invalid host")
+    return host, port
+
+
+def _valid_port(value: str, option: str) -> int:
+    try:
+        port = int(value)
+    except (TypeError, ValueError) as exc:
+        raise SystemExit(f"{option} requires a numeric port") from exc
+    if not 1 <= port <= 65535:
+        raise SystemExit(f"{option} port must be between 1 and 65535")
+    return port
+
+
+def _apply_web_host(host: str, environ: MutableMapping[str, str]) -> None:
+    environ["CIEL_RUNTIME_ROUTER_BIND_HOST"] = host
+    environ["CIEL_RUNTIME_ROUTER_CLIENT_HOST"] = (
+        "127.0.0.1" if host in _WILDCARD_HOSTS else host
+    )
+    if host in _WILDCARD_HOSTS or not _is_loopback(host):
+        environ["CIEL_RUNTIME_ROUTER_DEBUG_EXTERNAL"] = "1"
+
+
+def _is_loopback(host: str) -> bool:
+    if host.lower() == "localhost":
+        return True
+    try:
+        return ipaddress.ip_address(host).is_loopback
+    except ValueError:
+        return False
+
+
+def discover_tailscale_node(*, timeout: float = 2.0) -> TailscaleNode | None:
+    executable = shutil.which("tailscale")
+    if not executable:
+        return None
+    try:
+        completed = subprocess.run(
+            [executable, "status", "--json"],
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            check=False,
+        )
+        if completed.returncode != 0:
+            return None
+        payload = json.loads(completed.stdout)
+    except (OSError, subprocess.SubprocessError, json.JSONDecodeError):
+        return None
+    if str(payload.get("BackendState") or "").lower() != "running":
+        return None
+    self_node = payload.get("Self") if isinstance(payload.get("Self"), dict) else {}
+    addresses = self_node.get("TailscaleIPs") or payload.get("TailscaleIPs") or []
+    ipv4 = next((str(item) for item in addresses if _is_ipv4(item)), "")
+    dns_name = str(self_node.get("DNSName") or "").strip().rstrip(".")
+    if not ipv4 and not dns_name:
+        return None
+    return TailscaleNode(ipv4=ipv4, dns_name=dns_name, online=bool(self_node.get("Online", True)))
+
+
+def _is_ipv4(value: Any) -> bool:
+    try:
+        return ipaddress.ip_address(str(value)).version == 4
+    except ValueError:
+        return False
+
+
+def tailscale_https_url_for_target(
+    node: TailscaleNode,
+    router_port: int,
+    *,
+    timeout: float = 2.0,
+) -> str:
+    if not node.dns_name:
+        return ""
+    executable = shutil.which("tailscale")
+    if not executable:
+        return ""
+    try:
+        completed = subprocess.run(
+            [executable, "serve", "status", "--json"],
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+            check=False,
+        )
+        if completed.returncode != 0:
+            return ""
+        payload = json.loads(completed.stdout)
+    except (OSError, subprocess.SubprocessError, json.JSONDecodeError):
+        return ""
+    target = f"http://127.0.0.1:{router_port}"
+    web = payload.get("Web") if isinstance(payload.get("Web"), dict) else {}
+    tcp = payload.get("TCP") if isinstance(payload.get("TCP"), dict) else {}
+    for authority, config in web.items():
+        handlers = config.get("Handlers") if isinstance(config, dict) else {}
+        if not isinstance(handlers, dict):
+            continue
+        if not any(
+            isinstance(handler, dict) and str(handler.get("Proxy") or "").rstrip("/") == target
+            for handler in handlers.values()
+        ):
+            continue
+        port_text = str(authority).rsplit(":", 1)[-1]
+        port_config = tcp.get(port_text) if isinstance(tcp, dict) else None
+        if not isinstance(port_config, dict) or not port_config.get("HTTPS"):
+            continue
+        port = int(port_text) if port_text.isdigit() else 443
+        suffix = "" if port == 443 else f":{port}"
+        return f"https://{node.dns_name}{suffix}"
+    return ""
+
+
+def build_web_endpoint_report(
+    client_host: str,
+    bind_host: str,
+    router_port: int,
+) -> WebEndpointReport:
+    display_host = _url_host(client_host or "127.0.0.1")
+    local_url = f"http://{display_host}:{router_port}/"
+    node = discover_tailscale_node()
+    if node is None:
+        return WebEndpointReport(local_url=local_url)
+    exposed = bind_host in _WILDCARD_HOSTS or bind_host in {node.ipv4, node.dns_name}
+    tailscale_ip_url = f"http://{node.ipv4}:{router_port}/" if exposed and node.ipv4 else ""
+    https_url = tailscale_https_url_for_target(node, router_port)
+    command = ""
+    if not https_url and node.dns_name:
+        command = (
+            f"ciel-runtime --ca-web-port {router_port} --ca-tailscale-https={router_port}"
+        )
+    return WebEndpointReport(
+        local_url=local_url,
+        tailscale_ip_url=tailscale_ip_url,
+        tailscale_https_url=f"{https_url}/" if https_url else "",
+        tailscale_https_command=command,
+    )
+
+
+def configure_tailscale_https(router_port: int, https_port: int | None = None) -> list[str]:
+    node = discover_tailscale_node()
+    if node is None or not node.dns_name:
+        return ["Tailscale HTTPS was requested, but no online MagicDNS node was detected."]
+    executable = shutil.which("tailscale")
+    if not executable:
+        return ["Tailscale HTTPS was requested, but the tailscale CLI was not found."]
+    public_port = https_port or router_port
+    command = [
+        executable,
+        "serve",
+        f"--https={public_port}",
+        "--bg",
+        "--yes",
+        f"http://127.0.0.1:{router_port}",
+    ]
+    try:
+        completed = subprocess.run(
+            command,
+            capture_output=True,
+            text=True,
+            timeout=20.0,
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError) as exc:
+        return [f"Tailscale HTTPS setup failed: {type(exc).__name__}: {exc}"]
+    if completed.returncode != 0:
+        detail = (completed.stderr or completed.stdout or "unknown error").strip()
+        return [f"Tailscale HTTPS setup failed: {detail}"]
+    suffix = "" if public_port == 443 else f":{public_port}"
+    return [
+        f"Tailscale HTTPS: https://{node.dns_name}{suffix}/",
+        f"Tailscale Serve target: http://127.0.0.1:{router_port}",
+    ]
+
+
+def configure_requested_web_endpoints(
+    router_port: int,
+    client_host: str,
+    bind_host: str,
+    environ: MutableMapping[str, str] | None = None,
+    config: dict[str, Any] | None = None,
+) -> list[str]:
+    environment = os.environ if environ is None else environ
+    lines: list[str] = []
+    saved = web_backend_settings(config)
+    requested = (
+        str(environment.get("CIEL_RUNTIME_TAILSCALE_HTTPS") or "").lower() in _TRUE
+        or (saved.enabled and saved.tailscale_https)
+    )
+    if requested:
+        configured_port = str(environment.get("CIEL_RUNTIME_TAILSCALE_HTTPS_PORT") or "").strip()
+        https_port = _valid_port(configured_port, "CIEL_RUNTIME_TAILSCALE_HTTPS_PORT") if configured_port else None
+        lines.extend(configure_tailscale_https(router_port, https_port))
+    lines.extend(build_web_endpoint_report(client_host, bind_host, router_port).status_lines())
+    return lines
+
+
+def _url_host(host: str) -> str:
+    return f"[{host}]" if ":" in host and not host.startswith("[") else host
+
+
+__all__ = [
+    "TailscaleNode",
+    "WebBackendSettings",
+    "WebEndpointReport",
+    "apply_startup_web_options",
+    "build_web_endpoint_report",
+    "configure_requested_web_endpoints",
+    "configure_tailscale_https",
+    "discover_tailscale_node",
+    "load_saved_web_backend",
+    "tailscale_https_url_for_target",
+    "update_web_backend_config",
+    "web_backend_panel_rows",
+    "web_backend_settings",
+    "web_backend_summary",
+]