@oneciel-ai/ciel-runtime 0.2.22 → 0.2.23

package/ciel_runtime_support/responses_input_compatibility.py

@@ -26,6 +26,7 @@ _MISSING_TOOL_OUTPUT = re.compile(
     r"No tool output found for (?:custom|function) tool call ([A-Za-z0-9_-]+)",
     re.IGNORECASE,
 )
+_VALID_TOOL_NAME = re.compile(r"^[a-zA-Z0-9_-]+$")
 _TOOL_CALL_TYPES = frozenset({"function_call", "custom_tool_call"})
 _TOOL_OUTPUT_TYPES = frozenset(
     {"function_call_output", "custom_tool_call_output"}
@@ -68,6 +69,11 @@ def repair_replayed_response_items(body: dict[str, Any]) -> dict[str, Any]:
     keep ``call_id``, the tool name, arguments, output, and message content so
     the transcript remains usable.
 
+    Historical clients can also persist function names that the Responses API
+    does not accept, such as ``multi_tool_use.parallel``. Drop that call rather
+    than inventing a different tool name. The call/output balancing below then
+    drops its matching output by ``call_id``.
+
     IDs the router minted itself are treated the same way. They carry the right
     prefix, so the type check alone accepts them, but no backend ever stored
     them: OpenAI answers a replayed one with ``Item with id 'rs_..._0' not
@@ -94,6 +100,14 @@ def repair_replayed_response_items(body: dict[str, Any]) -> dict[str, Any]:
             continue
         item = value_item
         item_type = item.get("type")
+        item_name = item.get("name")
+        if (
+            item_type in _TOOL_CALL_TYPES
+            and item_name is not None
+            and not _VALID_TOOL_NAME.fullmatch(str(item_name))
+        ):
+            changed = True
+            continue
         item_id = str(item.get("id") or "")
         expected_prefix = OPENAI_RESPONSES_ITEM_ID_PREFIXES.get(str(item_type or ""))
         invalid_item_id = bool(item_id) and (

package/ciel_runtime_support/router_http.py

@@ -117,6 +117,7 @@ class RouterHttpPostEndpoints:
     runtime: Callable[..., bool]
     external_events_raw: Callable[[Any, str, bytes], bool] | None = None
     external_events_config: Callable[[Any, str, dict[str, Any]], bool] | None = None
+    usage: Callable[[Any, str, dict[str, Any]], bool] | None = None
 
 
 @dataclass(frozen=True, slots=True)
@@ -711,6 +712,8 @@ class RouterHttpHandler(BaseHTTPRequestHandler):
         path = parsed.path
         query = urllib.parse.parse_qs(parsed.query)
         cfg = services.core.load_config()
+        if path.startswith("/ca/usage/") and services.get.events(self, path, query):
+            return
         if services.core.reject_external(self, cfg):
             return
         endpoints = services.get
@@ -964,6 +967,8 @@ class RouterHttpHandler(BaseHTTPRequestHandler):
                         length,
                         body,
                     )
+                    if endpoints.usage is not None and endpoints.usage(self, path, body):
+                        return
                     if endpoints.external_events_config is not None and endpoints.external_events_config(self, path, body):
                         return
                     if endpoints.llm_config(self, path, body):

package/ciel_runtime_support/runtime_constants.py

@@ -87,7 +87,7 @@ OPENCODE_ENDPOINT_ALIASES = {
 }
 
 APP_NAME = "Ciel Runtime"
-VERSION = "0.2.22"
+VERSION = "0.2.23"
 CREDITS = "Credits: One Ciel LLC"
 PRELAUNCH_CANCEL = 10
 PRELAUNCH_LAUNCH_CODEX = 11

package/ciel_runtime_support/runtime_launch.py

@@ -18,7 +18,7 @@ from ciel_runtime_support.runtime_constants import (
     PRELAUNCH_LAUNCH_CODEX_APP_SERVER,
     ROUTED_COMPAT_PROMPT,
 )
-from ciel_runtime_support.runtime_paths import CONFIG_DIR, LOG_PATH, ROUTER_INSTANCE_DIR, WORKSPACE_STATE_DIR
+from ciel_runtime_support.runtime_paths import CONFIG_DIR, LOG_PATH, ROUTER_BASE, ROUTER_INSTANCE_DIR, WORKSPACE_STATE_DIR
 from ciel_runtime_support.web_endpoints import (
     current_web_workspace,
     web_backend_owned_by_workspace,
@@ -457,7 +457,13 @@ def run_claude(
     if should_attach_web_search(provider, cfg, web_search_override):
         mcp_config_paths.append(str(write_duckduckgo_mcp_config(cfg)))
     workspace_mcp_launch = (
-        workspace_mcp.prepare("claude", cfg, env) if workspace_mcp is not None else None
+        workspace_mcp.prepare(
+            "claude", cfg, env,
+            injected_servers={"ciel-runtime-router": {
+                "transport": "streamable_http", "url": f"{ROUTER_BASE}/ca/mcp",
+                "runtimes": ["claude"],
+            }} if llm_channel_delivery and web_backend_start_requested(cfg) else None,
+        ) if workspace_mcp is not None else None
     )
     if workspace_mcp_launch is not None:
         mcp_config_paths.extend(str(path) for path in workspace_mcp_launch.claude_config_paths)
@@ -961,6 +967,7 @@ def run_codex(
             cmd,
             env,
             wake_for_llm_delivery=channel_delivery_mode(cfg) == "llm",
+            channel_wake_display_body=True,
             synthetic_enter_bytes=codex_synthetic_enter,
             normalize_bare_cr_for_synthetic_enter=False,
             channel_wake_submit_retries=_codex_channel_wake_submit_retries(),

package/ciel_runtime_support/runtime_paths.py

@@ -155,6 +155,8 @@ LOG_LEVEL_PATH = CONFIG_DIR / "log-level"
 REQUEST_DUMP_PATH = ROUTER_INSTANCE_DIR / "requests.jsonl"
 RESPONSE_DUMP_PATH = ROUTER_INSTANCE_DIR / "responses.jsonl"
 USAGE_EVENTS_PATH = ROUTER_INSTANCE_DIR / "usage-events.jsonl"
+USAGE_LEDGER_PATH = WORKSPACE_STATE_DIR / "usage" / "usage.sqlite3"
+USAGE_API_KEY_PEPPER_PATH = WORKSPACE_STATE_DIR / "usage" / "api-key.pepper"
 SSE_TRACE_PATH = ROUTER_INSTANCE_DIR / "router-sse-trace.jsonl"
 SSE_LAST_PATH = ROUTER_INSTANCE_DIR / "router-last-sse.json"
 TOOL_CALL_LOG_PATH = ROUTER_INSTANCE_DIR / "tool-calls.jsonl"

package/ciel_runtime_support/tool_side_effect_dedupe.py

@@ -143,10 +143,13 @@ class ToolSideEffectDedupeService:
                 for block in blocks
                 if isinstance(block, dict) and block.get("type") == "tool_result"
             ]
-            has_new_intent = isinstance(content, str) and bool(content.strip())
-            has_new_intent = has_new_intent or any(
-                not isinstance(block, dict) or block.get("type") != "tool_result"
-                for block in blocks
+            runtime_control = bool(message.get("ciel_runtime_control"))
+            has_new_intent = not runtime_control and (
+                (isinstance(content, str) and bool(content.strip()))
+                or any(
+                    not isinstance(block, dict) or block.get("type") != "tool_result"
+                    for block in blocks
+                )
             )
             if has_new_intent:
                 completed.clear()

package/ciel_runtime_support/transcript_delta_delivery.py

@@ -0,0 +1,333 @@
+"""Incremental transcript delivery to one configured outbound HTTP endpoint."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import hashlib
+import json
+from pathlib import Path
+import threading
+import time
+from typing import Any
+import urllib.error
+import urllib.request
+
+from .remote_instructions import expand_environment_references
+
+
+@dataclass(frozen=True, slots=True)
+class TranscriptDeliverySettings:
+    enabled: bool
+    url: str
+    authorization: str
+    timeout_seconds: float
+    poll_interval_seconds: float
+    max_batch_bytes: int
+    start_mode: str
+
+    @classmethod
+    def from_config(cls, config: dict[str, Any]) -> "TranscriptDeliverySettings":
+        raw = config.get("transcript_events")
+        values = raw if isinstance(raw, dict) else {}
+        start_mode = str(values.get("start_mode") or "tail").strip().lower()
+        if start_mode not in {"tail", "beginning"}:
+            start_mode = "tail"
+        return cls(
+            enabled=bool(values.get("enabled", False)),
+            url=str(values.get("url") or "").strip(),
+            authorization=str(values.get("authorization") or "").strip(),
+            timeout_seconds=max(0.1, min(30.0, float(values.get("timeout_seconds") or 5))),
+            poll_interval_seconds=max(
+                0.1, min(60.0, float(values.get("poll_interval_ms") or 1000) / 1000.0)
+            ),
+            max_batch_bytes=max(
+                1024, min(16_777_216, int(values.get("max_batch_bytes") or 1_048_576))
+            ),
+            start_mode=start_mode,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class TranscriptDeliveryPorts:
+    load_config: Callable[[], dict[str, Any]]
+    latest_transcript: Callable[[], Path | None]
+    scope: Callable[[], dict[str, Any]]
+    log: Callable[[str, str], None]
+    epoch: Callable[[], float] = time.time
+
+
+class TranscriptDeltaDeliveryService:
+    """Tail complete JSONL records and commit offsets only after HTTP success."""
+
+    def __init__(
+        self,
+        cursor_path: Path,
+        workspace_id: str,
+        ports: TranscriptDeliveryPorts,
+    ) -> None:
+        self.cursor_path = cursor_path
+        self.workspace_id = str(workspace_id or "workspace")
+        self.ports = ports
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+        self._lock = threading.Lock()
+        self._last_error = ""
+        self._last_error_at = 0.0
+
+    def start(self) -> None:
+        with self._lock:
+            if self._thread is not None and self._thread.is_alive():
+                return
+            self._stop.clear()
+            self._thread = threading.Thread(
+                target=self._run,
+                name="ciel-transcript-delivery",
+                daemon=True,
+            )
+            self._thread.start()
+
+    def stop(self, timeout: float = 2.0) -> None:
+        self._stop.set()
+        thread = self._thread
+        if thread is not None and thread is not threading.current_thread():
+            thread.join(max(0.0, timeout))
+
+    def poll_once(self) -> bool:
+        settings = TranscriptDeliverySettings.from_config(self.ports.load_config())
+        if not settings.enabled or not settings.url:
+            return False
+        path = self.ports.latest_transcript()
+        if path is None:
+            return False
+        try:
+            path = path.resolve()
+            size = path.stat().st_size
+        except OSError:
+            return False
+        scope = self.ports.scope()
+        session_id = str(scope.get("session_id") or path.stem)
+        runtime = str(scope.get("runtime") or "runtime")
+        destination_key = self._destination_key(settings.url, path)
+        cursors = self._load_cursors()
+        destinations = cursors.setdefault("destinations", {})
+        current = destinations.get(destination_key)
+        if not isinstance(current, dict):
+            offset = self._initial_offset(settings, scope, path)
+            destinations[destination_key] = self._cursor_record(path, offset)
+            self._save_cursors(cursors)
+            if offset >= size:
+                return False
+            current = destinations[destination_key]
+        offset = max(0, int(current.get("offset") or 0))
+        rotated = size < offset
+        if rotated:
+            offset = 0
+        payload = self._read_complete_batch(path, offset, settings.max_batch_bytes)
+        if not payload:
+            if rotated:
+                destinations[destination_key] = self._cursor_record(path, offset)
+                self._save_cursors(cursors)
+            return False
+        end_offset = offset + len(payload)
+        event = self._cloud_event(
+            runtime=runtime,
+            session_id=session_id,
+            path=path,
+            start_offset=offset,
+            end_offset=end_offset,
+            content=payload,
+            rotated=rotated,
+        )
+        if not self._post(settings, event):
+            return False
+        destinations[destination_key] = self._cursor_record(path, end_offset)
+        self._save_cursors(cursors)
+        self.ports.log(
+            "INFO",
+            "transcript_delta_delivered "
+            f"runtime={runtime} session={session_id} bytes={len(payload)} "
+            f"offset={offset}->{end_offset}",
+        )
+        self._last_error = ""
+        return True
+
+    @staticmethod
+    def _initial_offset(
+        settings: TranscriptDeliverySettings, scope: dict[str, Any], path: Path
+    ) -> int:
+        if settings.start_mode == "beginning":
+            return 0
+        boundary_path = scope.get("turn_scan_path")
+        if boundary_path is None:
+            return 0
+        try:
+            if Path(boundary_path).resolve() != path:
+                return 0
+        except (OSError, TypeError, ValueError):
+            return 0
+        return max(0, int(scope.get("turn_scan_offset") or 0))
+
+    def _run(self) -> None:
+        while not self._stop.is_set():
+            interval = 1.0
+            try:
+                config = self.ports.load_config()
+                settings = TranscriptDeliverySettings.from_config(config)
+                interval = settings.poll_interval_seconds
+                if settings.enabled and settings.url:
+                    self.poll_once()
+            except Exception as exc:
+                self._report_error(f"poll {type(exc).__name__}: {exc}")
+            self._stop.wait(interval)
+
+    def _load_cursors(self) -> dict[str, Any]:
+        try:
+            value = json.loads(self.cursor_path.read_text(encoding="utf-8"))
+        except (OSError, ValueError, TypeError):
+            value = {}
+        if not isinstance(value, dict):
+            value = {}
+        if not isinstance(value.get("destinations"), dict):
+            value["destinations"] = {}
+        value["version"] = 1
+        return value
+
+    def _save_cursors(self, cursors: dict[str, Any]) -> None:
+        self.cursor_path.parent.mkdir(parents=True, exist_ok=True)
+        temporary = self.cursor_path.with_name(
+            f"{self.cursor_path.name}.{threading.get_ident()}.tmp"
+        )
+        temporary.write_text(
+            json.dumps(cursors, ensure_ascii=False, separators=(",", ":")),
+            encoding="utf-8",
+        )
+        temporary.replace(self.cursor_path)
+
+    def _cursor_record(self, path: Path, offset: int) -> dict[str, Any]:
+        return {
+            "path": str(path),
+            "offset": max(0, int(offset)),
+            "updated_at": self.ports.epoch(),
+        }
+
+    @staticmethod
+    def _destination_key(url: str, path: Path) -> str:
+        return hashlib.sha256(f"{url}\0{path}".encode("utf-8")).hexdigest()
+
+    def _read_complete_batch(self, path: Path, offset: int, limit: int) -> bytes:
+        try:
+            with path.open("rb") as stream:
+                stream.seek(offset)
+                data = stream.read(limit + 1)
+        except OSError as exc:
+            self._report_error(f"read {type(exc).__name__}: {exc}")
+            return b""
+        if not data:
+            return b""
+        bounded = data[:limit]
+        complete_end = bounded.rfind(b"\n")
+        if complete_end < 0:
+            if len(data) > limit:
+                self._report_error(
+                    f"record exceeds max_batch_bytes={limit} path={path.name} offset={offset}"
+                )
+            return b""
+        return bounded[: complete_end + 1]
+
+    def _cloud_event(
+        self,
+        *,
+        runtime: str,
+        session_id: str,
+        path: Path,
+        start_offset: int,
+        end_offset: int,
+        content: bytes,
+        rotated: bool,
+    ) -> dict[str, Any]:
+        digest = hashlib.sha256(content).hexdigest()
+        event_id = hashlib.sha256(
+            (
+                f"{self.workspace_id}\0{session_id}\0{path}\0"
+                f"{start_offset}\0{end_offset}\0{digest}"
+            ).encode("utf-8")
+        ).hexdigest()
+        return {
+            "specversion": "1.0",
+            "id": event_id,
+            "source": f"urn:ciel-runtime:workspace:{self.workspace_id}",
+            "type": "ai.oneciel.ciel-runtime.transcript.delta",
+            "subject": session_id,
+            "time": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
+            "datacontenttype": "application/json",
+            "data": {
+                "workspace_id": self.workspace_id,
+                "runtime": runtime,
+                "session_id": session_id,
+                "transcript_name": path.name,
+                "start_offset": start_offset,
+                "end_offset": end_offset,
+                "content_sha256": digest,
+                "record_count": content.count(b"\n"),
+                "rotated": rotated,
+                "format": "jsonl",
+                "content": content.decode("utf-8", errors="replace"),
+            },
+        }
+
+    def _post(
+        self, settings: TranscriptDeliverySettings, event: dict[str, Any]
+    ) -> bool:
+        body = json.dumps(event, ensure_ascii=False, separators=(",", ":")).encode(
+            "utf-8"
+        )
+        headers = {
+            "Content-Type": "application/cloudevents+json",
+            "Accept": "application/json",
+            "User-Agent": "ciel-runtime-transcript-delivery/1",
+            "Idempotency-Key": str(event["id"]),
+        }
+        authorization, missing = expand_environment_references(settings.authorization)
+        if missing:
+            self._report_error(
+                "authorization environment variable(s) missing: " + ", ".join(missing)
+            )
+            return False
+        if authorization:
+            headers["Authorization"] = authorization
+        request = urllib.request.Request(
+            settings.url, data=body, headers=headers, method="POST"
+        )
+        try:
+            with urllib.request.urlopen(
+                request, timeout=settings.timeout_seconds
+            ) as response:
+                status = int(getattr(response, "status", 200) or 200)
+                response.read(4096)
+        except urllib.error.HTTPError as exc:
+            self._report_error(f"HTTP {exc.code} from transcript endpoint")
+            return False
+        except (OSError, urllib.error.URLError, TimeoutError) as exc:
+            self._report_error(f"delivery {type(exc).__name__}: {exc}")
+            return False
+        if not 200 <= status < 300:
+            self._report_error(f"HTTP {status} from transcript endpoint")
+            return False
+        return True
+
+    def _report_error(self, message: str) -> None:
+        now = self.ports.epoch()
+        if message == self._last_error and now - self._last_error_at < 30.0:
+            return
+        self._last_error = message
+        self._last_error_at = now
+        self.ports.log("WARN", f"transcript_delta_delivery_failed {message}")
+
+
+__all__ = [
+    "TranscriptDeliveryPorts",
+    "TranscriptDeliverySettings",
+    "TranscriptDeltaDeliveryService",
+]

package/ciel_runtime_support/usage_events.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import json
 import threading
 import time
+import uuid
 from dataclasses import asdict, dataclass
 from pathlib import Path
 from typing import Callable, Protocol
@@ -20,17 +21,57 @@ class UsageEvent:
     protocol: str = ""
     status: str = "completed"
     timestamp: float = 0.0
+    event_id: str = ""
+    runtime: str = ""
+    workspace_id: str = ""
+    session_id: str = ""
+    turn_id: str = ""
+    cache_read_input_tokens: int = 0
+    cache_write_input_tokens: int = 0
+    input_tokens_total: int = 0
+    reasoning_output_tokens: int = 0
+    request_started_at: float = 0.0
+    request_completed_at: float = 0.0
+    duration_ms: float = 0.0
+    usage_source: str = "router_response"
+    is_estimated: bool = False
+    is_incomplete: bool = False
 
     def normalized(self, clock: Callable[[], float] = time.time) -> UsageEvent:
+        timestamp = float(self.timestamp or clock())
+        completed_at = float(self.request_completed_at or timestamp)
+        started_at = float(self.request_started_at or completed_at)
+        input_tokens = max(0, int(self.input_tokens or 0))
+        cache_read = max(0, int(self.cache_read_input_tokens or 0))
+        cache_write = max(0, int(self.cache_write_input_tokens or 0))
         return UsageEvent(
             provider=str(self.provider or "").strip(),
             model=str(self.model or "").strip(),
-            input_tokens=max(0, int(self.input_tokens or 0)),
+            input_tokens=input_tokens,
             output_tokens=max(0, int(self.output_tokens or 0)),
             request_id=str(self.request_id or "").strip(),
             protocol=str(self.protocol or "").strip(),
             status=str(self.status or "completed").strip() or "completed",
-            timestamp=float(self.timestamp or clock()),
+            timestamp=timestamp,
+            event_id=str(self.event_id or uuid.uuid4().hex).strip(),
+            runtime=str(self.runtime or "").strip(),
+            workspace_id=str(self.workspace_id or "").strip(),
+            session_id=str(self.session_id or "").strip(),
+            turn_id=str(self.turn_id or "").strip(),
+            cache_read_input_tokens=cache_read,
+            cache_write_input_tokens=cache_write,
+            input_tokens_total=max(
+                0,
+                int(self.input_tokens_total or (input_tokens + cache_read + cache_write)),
+            ),
+            reasoning_output_tokens=max(0, int(self.reasoning_output_tokens or 0)),
+            request_started_at=started_at,
+            request_completed_at=completed_at,
+            duration_ms=max(0.0, float(self.duration_ms or 0.0)),
+            usage_source=str(self.usage_source or "router_response").strip()
+            or "router_response",
+            is_estimated=bool(self.is_estimated),
+            is_incomplete=bool(self.is_incomplete),
         )
 
 
@@ -78,8 +119,9 @@ class CompositeUsageEventSink:
         self._sinks = sinks
 
     def record(self, event: UsageEvent) -> None:
+        normalized = event.normalized()
         for sink in self._sinks:
-            sink.record(event)
+            sink.record(normalized)
 
 
 def summarize_usage(events: list[UsageEvent]) -> dict[tuple[str, str], dict[str, int]]: