@inline-chat/hermes-agent-adapter 1.0.1 → 1.1.0

package/plugin/inline/tools.py

@@ -12,6 +12,15 @@ from typing import Any, Dict, Iterable, Optional
 
 from .message_actions import build_inline_agent_action_id
 from .profile import hermes_home, inline_env, multiplex_active
+from .widgets import (
+    MAX_MESSAGE_TEXT_UTF16_UNITS,
+    MAX_MESSAGE_TEXT_UTF8_BYTES,
+    WidgetError,
+    build_widget_markdown,
+    has_widget_markup,
+    scan_widgets,
+    utf16_len,
+)
 
 try:
     from tools.registry import tool_error, tool_result
@@ -71,12 +80,25 @@ INLINE_PLATFORM_GUIDANCE = (
     "do not treat thread ids as reply/quote message ids. Supported Markdown is "
     "emphasis, inline or fenced code, links, headings, lists or checklists, "
     "quotes, tables, separators, and HTTP(S) images. Do not rely on "
-    "strikethrough, footnotes, or arbitrary HTML; unsupported or ambiguous "
+    "strikethrough, footnotes, or arbitrary HTML outside widgets; unsupported or ambiguous "
     "incomplete syntax remains visible text. Use ordinary Markdown "
     "tables for tabular data, never fenced-code tables. For collapsible work, "
     "use <details open>, then <summary>Title</summary>, body Markdown, and "
     "</details>; put kind=\"progress\" on summary only while work is in progress. "
-    "Use <footer>Attribution or brief metadata</footer> for a short footer."
+    "Use <footer>Attribution or brief metadata</footer> for a short footer. "
+    "Widgets: one sandboxed HTML block per message, only when Markdown or tables can't show it "
+    "(charts, diagrams, mockups, comparison grids, small pickers); keep prose outside it. Prefer "
+    "the inline tool's send_widget action; in a reply, a block on its own lines works too: "
+    "<iframe title=\"Name\" id=\"name\" height=\"420\">HTML</iframe> (one widget per message; "
+    "height 80-2000 px, about the content height at 640px wide). The sandbox has no network "
+    "(no fetch, XHR, WebSocket or remote images; use data: URIs); scripts may be inline or from "
+    "cdn.jsdelivr.net, cdnjs.cloudflare.com, unpkg.com or esm.sh. Avoid vh units, position: fixed "
+    "and nested frames; fit 320-640px widths. Theme CSS variables: --background --foreground "
+    "--muted --muted-foreground --card --border --primary --primary-foreground --accent --success "
+    "--warning --destructive --radius --font-sans --font-mono; classes .card .grid .row .badge "
+    ".btn .btn-primary .btn-ghost .table .muted .small. After a viewer's click, scripts can call "
+    "inline.copy(text), inline.sendPrompt(text), inline.action(name, payload) (reaches you as a "
+    "widget call turn), inline.openLink(url), and inline.setState(obj) with inline.state."
 )
 
 # Where each turn's triggering message physically lives, and who sent it. Hermes tells tools
@@ -129,7 +151,8 @@ _sidecars: Dict[str, Dict[str, Any]] = {}
 _settings_owners: Dict[str, Any] = {}
 
 _ACTION_MANIFEST = [
-    ("send_message", "(chat_id?|user_id?, text, buttons?)", "Send a message, including optional agent-owned buttons."),
+    ("send_message", "(chat_id?|user_id?, text, buttons?, reply_to_message_id?)", "Send a message, including optional agent-owned buttons."),
+    ("send_widget", "(chat_id?|user_id?, title, html, widget_id?, height?, text?, reply_to_message_id?)", "Send one sandboxed HTML widget, with optional lead text above it."),
     ("get_chat", "(chat_id?)", "Get metadata for a chat or reply thread, including whether you follow it."),
     ("set_follow_mode", "(chat_id?, mode)", "Follow a thread to join its conversation, or unfollow it to answer only mentions and replies. Applies to you only; mode is following or unfollowed."),
     ("get_messages", "(chat_id?, message_ids)", "Fetch specific Inline messages by ID."),
@@ -238,6 +261,7 @@ def tool_context_prompt(
     lines = [
         "- The `inline` tool is available for Inline history, search, exact message lookup, reactions, pins, and explicit thread management.",
         "- Return normal replies as text. Use inline send_message only to create a message with action buttons or to send into a different destination.",
+        "- For a chart, diagram, mockup or picker that Markdown can't show, use inline send_widget (then reply NO_REPLY unless more text is needed). A widget call from a viewer arrives as a turn naming widget_id, call_name and untrusted payload_json.",
         "- An Inline action-button turn names its source message. A normal response replaces that message; omitted buttons clear the old buttons. Use edit_message with buttons to replace them explicitly.",
         "- If edit_message already finalized the source action card during this turn, return NO_REPLY so the automatic final response does not overwrite that edit.",
     ]
@@ -270,6 +294,7 @@ INLINE_TOOL_SCHEMA = {
         + "\n\n"
         "When chat_id is omitted, the tool uses the current Inline reply thread if present, otherwise the current chat. "
         "Return normal assistant replies as text. Use send_message only for a button card or a different destination. "
+        "Use send_widget for one sandboxed HTML widget (see the Inline platform guidance); it is sent whole, and Inline's reason is returned if it rejects the widget. "
         "To hand work to another bot, mention it in your normal reply; its answer comes back to you as a turn that doesn't need a response, and asking it again takes a new mention. "
         "Agent-owned button presses return as ordinary turns, and the normal response replaces the source card while clearing omitted buttons. "
         "Use create_thread on the current triggering message when someone asks to continue in a thread; your answer then goes into it, so don't also post it in the chat, and it starts with a short summary of the conversation so far. "
@@ -300,7 +325,11 @@ INLINE_TOOL_SCHEMA = {
                 "type": "string",
                 "description": "Parent message ID for create_thread. Defaults to the triggering message when available.",
             },
-            "title": {"type": "string", "description": "Thread title. Required for create_chat, optional for create_thread, and the new name for update_chat."},
+            "title": {"type": "string", "description": "Thread title. Required for create_chat, optional for create_thread, and the new name for update_chat. For send_widget, the widget's title."},
+            "html": {"type": "string", "description": "send_widget: the widget's HTML body (no <iframe> wrapper, no </iframe>)."},
+            "widget_id": {"type": "string", "description": "send_widget: widget id, 1-64 of A-Z a-z 0-9 _ . : - (default widget). Widget calls name it."},
+            "height": {"type": "integer", "minimum": 80, "maximum": 2000, "description": "send_widget: frame height in CSS px, about the content height at 640px wide."},
+            "reply_to_message_id": {"type": "string", "description": "send_message/send_widget: message ID to reply to."},
             "name": {"type": "string", "description": "Agent name for create_agent."},
             "agent_id": {"type": "string", "description": "Globally unique Inline Agent ID for get_agent, update_agent, or delete_agent."},
             "handle": {"type": "string", "description": "Optional Agent handle."},
@@ -320,7 +349,7 @@ INLINE_TOOL_SCHEMA = {
                 "description": "Whether create_chat is visible to the parent space. Defaults to false and requires space_id when true.",
             },
             "query": {"type": "string", "description": "Search query for search_messages."},
-            "text": {"type": "string", "description": "Message text for send_message or edit_message."},
+            "text": {"type": "string", "description": "Message text for send_message or edit_message; optional lead text above a send_widget widget."},
             "buttons": {
                 "type": "array",
                 "maxItems": _MAX_ACTION_ROWS,
@@ -387,7 +416,13 @@ def _handle_inline_tool(args: Dict[str, Any], **_: Any) -> str:
         if action == "create_thread":
             return tool_result({"success": True, "action": action, "result": _create_thread(args)})
         path, body = _request_for_action(action, args)
-        response = _sidecar_call(path, body)
+        try:
+            response = _sidecar_call(path, body)
+        except InlineToolError as exc:
+            if action == "send_widget" and exc.error_kind == "bad_format":
+                # The server names the widget rule the message broke; the model can fix and resend.
+                raise InlineToolError(f"Inline rejected the widget: {exc}", exc.error_kind) from exc
+            raise
         result = _compact_result(action, response.get("result") or {})
         return tool_result({"success": True, "action": action, "result": result})
     except InlineToolError as exc:
@@ -396,14 +431,37 @@ def _handle_inline_tool(args: Dict[str, Any], **_: Any) -> str:
 
 def _request_for_action(action: str, args: Dict[str, Any]) -> tuple[str, Dict[str, Any]]:
     if action == "send_message":
+        parse_markdown = _bool(args.get("parse_markdown"), True)
         body = {
             "target": _target(args),
-            "text": _required_str(args, "text", max_chars=_MAX_TEXT_CHARS),
-            "parseMarkdown": _bool(args.get("parse_markdown"), True),
+            "text": _message_text(args, parse_markdown),
+            "parseMarkdown": parse_markdown,
         }
         actions = _message_actions(args)
         if actions is not None:
             body["actions"] = actions
+        reply_to = _inline_id(args.get("reply_to_message_id"))
+        if reply_to:
+            body["replyToMsgId"] = reply_to
+        return "/send", body
+
+    if action == "send_widget":
+        if not _str(args.get("title")):
+            raise InlineToolError("title is required", "bad_format")
+        try:
+            text = build_widget_markdown(
+                html=str(args.get("html") or ""),
+                title=_str(args.get("title")),
+                widget_id=_str(args.get("widget_id")),
+                height=args.get("height"),
+                text=str(args.get("text") or ""),
+            )
+        except WidgetError as exc:
+            raise InlineToolError(str(exc), "bad_format") from exc
+        body = {"target": _target(args), "text": text, "parseMarkdown": True}
+        reply_to = _inline_id(args.get("reply_to_message_id"))
+        if reply_to:
+            body["replyToMsgId"] = reply_to
         return "/send", body
 
     if action == "get_chat":
@@ -437,11 +495,12 @@ def _request_for_action(action: str, args: Dict[str, Any]) -> tuple[str, Dict[st
         return "/search", body
 
     if action == "edit_message":
+        parse_markdown = _bool(args.get("parse_markdown"), True)
         body = {
             "target": _target(args),
             "messageId": _required_id(args, "message_id"),
-            "text": _required_str(args, "text", max_chars=_MAX_TEXT_CHARS),
-            "parseMarkdown": _bool(args.get("parse_markdown"), True),
+            "text": _message_text(args, parse_markdown),
+            "parseMarkdown": parse_markdown,
         }
         actions = _message_actions(args)
         if actions is not None:
@@ -895,7 +954,7 @@ def _message_actions(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
 
 
 def _compact_result(action: str, result: Dict[str, Any]) -> Dict[str, Any]:
-    if action in {"send_message", "edit_message"}:
+    if action in {"send_message", "send_widget", "edit_message"}:
         return {"messageId": _str(result.get("messageId"))}
     if action == "get_chat":
         return _compact_chat(result)
@@ -1166,6 +1225,18 @@ def _required_str(args: Dict[str, Any], key: str, *, max_chars: int) -> str:
     return _truncate(value, max_chars)
 
 
+def _message_text(args: Dict[str, Any], parse_markdown: bool) -> str:
+    """Message text, shortened as before unless it holds a widget, which is sent whole or refused."""
+    value = _str(args.get("text"))
+    if not value:
+        raise InlineToolError("text is required", "bad_format")
+    if not parse_markdown or not has_widget_markup(value) or not scan_widgets(value).widgets:
+        return _truncate(value, _MAX_TEXT_CHARS)
+    if utf16_len(value) > MAX_MESSAGE_TEXT_UTF16_UNITS or len(value.encode("utf-8")) > MAX_MESSAGE_TEXT_UTF8_BYTES:
+        raise InlineToolError("message with a widget exceeds Inline's 100,000 character limit", "bad_format")
+    return value
+
+
 def _required_id(args: Dict[str, Any], key: str) -> str:
     value = _inline_id(args.get(key))
     if not value:

package/plugin/inline/widgets.py

@@ -0,0 +1,267 @@
+"""Inline message widgets: AFM ``<iframe>`` previews in bot Markdown.
+
+The server turns one ``<iframe title=… id=… height=…>HTML</iframe>`` block in a
+bot's Markdown into a sandboxed widget. These helpers mirror its parsing rules
+so outbound text is never split inside a widget, and build the Markdown for the
+``send_widget`` tool action.
+"""
+from __future__ import annotations
+
+import html as _html
+import re
+from typing import Any, Callable, List, NamedTuple, Optional
+
+
+# Server limits (server/src/modules/message/messageWidgets.ts, messageTextLimits.ts).
+MAX_MESSAGE_TEXT_UTF16_UNITS = 100_000
+MAX_MESSAGE_TEXT_UTF8_BYTES = 400_000
+MAX_WIDGET_HTML_BYTES = 256 * 1024
+MAX_WIDGET_TITLE_CHARS = 256
+MIN_WIDGET_HEIGHT = 80
+MAX_WIDGET_HEIGHT = 2000
+
+_WIDGET_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,64}$")
+# Same shapes as the server: the opener starts a line; the first closer ends the payload.
+_OPENER_RE = re.compile(r"""<iframe(?=[\s>])((?:[^"'<>]|"[^"]*"|'[^']*')*)>""", re.IGNORECASE)
+_CLOSER_RE = re.compile(r"</iframe\s*>", re.IGNORECASE)
+_CLOSER_PREFIX_RE = re.compile(r"</iframe", re.IGNORECASE)
+_OPENER_PREFIX_RE = re.compile(r"<iframe", re.IGNORECASE)
+_OPENING_FENCE_RE = re.compile(r"( {0,3})(`{3,}|~{3,})(.*)")
+_CLOSING_FENCE_RE = re.compile(r" {0,3}(`+|~+)[ \t]*")
+_ATTRIBUTE_RE = re.compile(r"""([a-zA-Z_:][-a-zA-Z0-9_:.]*)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?""")
+
+
+class WidgetSpan(NamedTuple):
+    """A complete widget block from its opener line start to the end of its closing line."""
+    start: int
+    end: int
+    title: str
+
+
+class WidgetScan(NamedTuple):
+    widgets: List[WidgetSpan]
+    # Offset of an opener whose closing tag has not arrived (a streaming preview), else None.
+    unclosed_at: Optional[int]
+
+
+class WidgetError(ValueError):
+    """A widget problem the model can fix."""
+
+
+def utf16_len(text: str) -> int:
+    return len(text.encode("utf-16-le")) // 2
+
+
+def _read_line(text: str, start: int) -> tuple[int, int]:
+    """(content_end, next_line_start) for the line starting at ``start``."""
+    newline = text.find("\n", start)
+    raw_end = newline if newline >= 0 else len(text)
+    content_end = raw_end - 1 if raw_end > start and text[raw_end - 1] == "\r" else raw_end
+    return content_end, (raw_end + 1 if raw_end < len(text) else len(text))
+
+
+def _opener_title(attributes: str) -> str:
+    for match in _ATTRIBUTE_RE.finditer(attributes):
+        if match.group(1).lower() != "title":
+            continue
+        value = next((group for group in match.groups()[1:] if group is not None), "")
+        return " ".join(_html.unescape(value).split())[:MAX_WIDGET_TITLE_CHARS]
+    return ""
+
+
+def scan_widgets(text: str) -> WidgetScan:
+    """Find widget blocks as the server does: outside code fences, with the opener at a line start."""
+    widgets: List[WidgetSpan] = []
+    fence: Optional[tuple[str, int]] = None
+    cursor = 0
+    while cursor < len(text):
+        content_end, next_line = _read_line(text, cursor)
+        line = text[cursor:content_end]
+        if fence:
+            closing = _CLOSING_FENCE_RE.fullmatch(line)
+            if closing and closing.group(1)[0] == fence[0] and len(closing.group(1)) >= fence[1]:
+                fence = None
+            cursor = next_line
+            continue
+        opening = _OPENING_FENCE_RE.fullmatch(line)
+        if opening and not (opening.group(2)[0] == "`" and "`" in opening.group(3)):
+            fence = (opening.group(2)[0], len(opening.group(2)))
+            cursor = next_line
+            continue
+        opener = _OPENER_RE.match(line)
+        if not opener:
+            cursor = next_line
+            continue
+        closer = _CLOSER_RE.search(text, cursor + opener.end())
+        if not closer:
+            return WidgetScan(widgets, cursor)
+        closing_start = text.rfind("\n", 0, closer.start()) + 1
+        closing_end, after_closing = _read_line(text, closing_start)
+        widgets.append(WidgetSpan(cursor, closing_end, _opener_title(opener.group(1) or "")))
+        cursor = after_closing
+    return WidgetScan(widgets, None)
+
+
+def has_widget_markup(text: str) -> bool:
+    """Cheap pre-check before a full scan."""
+    return _OPENER_PREFIX_RE.search(text) is not None
+
+
+def split_widget_aware(
+    text: str,
+    max_length: int,
+    split_plain: Callable[[str, int], List[str]],
+    len_fn: Callable[[str], int] = len,
+) -> Optional[List[str]]:
+    """Split ``text`` into messages without cutting a widget or putting two in one message.
+
+    Returns None when the text has no complete widget, so the caller keeps its plain splitter.
+    A widget block longer than ``max_length`` becomes its own message (the server allows
+    100,000 UTF-16 units); prose around widgets is split with ``split_plain``.
+    """
+    if not has_widget_markup(text):
+        return None
+    scan = scan_widgets(text)
+    if not scan.widgets:
+        return None
+    if len(scan.widgets) == 1 and len_fn(text) <= max_length:
+        return [text]
+
+    segments: List[tuple[bool, str]] = []
+    cursor = 0
+    for widget in scan.widgets:
+        segments.append((False, text[cursor:widget.start]))
+        segments.append((True, text[widget.start:widget.end]))
+        cursor = widget.end
+    segments.append((False, text[cursor:]))
+
+    chunks: List[str] = []
+    current = ""
+    current_has_widget = False
+
+    def joined(part: str) -> str:
+        return f"{current}\n\n{part}" if current else part
+
+    def flush() -> None:
+        nonlocal current, current_has_widget
+        if current.strip():
+            chunks.append(current)
+        current = ""
+        current_has_widget = False
+
+    for is_widget, raw in segments:
+        part = raw.strip("\r\n")
+        if not part.strip():
+            continue
+        if is_widget:
+            if current and (current_has_widget or len_fn(joined(part)) > max_length):
+                flush()
+            current = joined(part)
+            current_has_widget = True
+            continue
+        if len_fn(joined(part)) <= max_length:
+            current = joined(part)
+            continue
+        flush()
+        pieces = [piece for piece in split_plain(part, max_length) if piece.strip()]
+        chunks.extend(pieces[:-1])
+        current = pieces[-1] if pieces else ""
+    flush()
+    return chunks or [text]
+
+
+def streaming_preview(text: str) -> str:
+    """Text safe to show while a reply streams: hide a widget whose closing tag hasn't arrived."""
+    if not has_widget_markup(text):
+        return text
+    scan = scan_widgets(text)
+    if scan.unclosed_at is None:
+        return text
+    return text[:scan.unclosed_at].rstrip() or "…"
+
+
+def replace_widgets(text: str, replacement: Callable[[WidgetSpan], str]) -> str:
+    if not has_widget_markup(text):
+        return text
+    scan = scan_widgets(text)
+    out: List[str] = []
+    cursor = 0
+    for widget in scan.widgets:
+        out.append(text[cursor:widget.start])
+        out.append(replacement(widget))
+        cursor = widget.end
+    out.append(text[cursor:])
+    return "".join(out)
+
+
+def without_widgets(text: str) -> str:
+    """Replace each widget with a short notice, for a fallback after the server rejected one."""
+    def notice(widget: WidgetSpan) -> str:
+        return f'[Widget "{widget.title or "HTML preview"}" could not be displayed]'
+
+    replaced = replace_widgets(text, notice)
+    scan = scan_widgets(replaced)
+    if scan.unclosed_at is not None:
+        replaced = replaced[:scan.unclosed_at].rstrip() + "\n\n[Widget could not be displayed]"
+    return replaced
+
+
+def _escape_attribute(value: str) -> str:
+    # The server decodes exactly &quot; &amp; &lt; &gt; &#39; &apos; in attribute values.
+    return value.replace("&", "&amp;").replace('"', "&quot;").replace("<", "&lt;").replace(">", "&gt;")
+
+
+def _height(value: Any) -> Optional[int]:
+    if value is None or value == "":
+        return None
+    if isinstance(value, bool):
+        raise WidgetError("height must be a whole number of CSS pixels from 80 to 2000")
+    text = str(value).strip().lower().removesuffix("px")
+    if not text.isdigit():
+        raise WidgetError("height must be a whole number of CSS pixels from 80 to 2000")
+    height = int(text)
+    if height < MIN_WIDGET_HEIGHT or height > MAX_WIDGET_HEIGHT:
+        raise WidgetError("height must be from 80 to 2000 CSS pixels")
+    return height
+
+
+def build_widget_markdown(
+    *,
+    html: str,
+    title: Optional[str] = None,
+    widget_id: Optional[str] = None,
+    height: Any = None,
+    text: Optional[str] = None,
+) -> str:
+    """Markdown for one widget message: optional lead text, then the ``<iframe>`` block."""
+    body = str(html or "").strip("\r\n")
+    if not body.strip():
+        raise WidgetError("html is required")
+    if _CLOSER_PREFIX_RE.search(body):
+        raise WidgetError("html must not contain </iframe: it would end the widget early (nested iframes are not allowed)")
+    if len(body.encode("utf-8")) > MAX_WIDGET_HTML_BYTES:
+        raise WidgetError("html exceeds 256 KiB")
+    attributes: List[str] = []
+    clean_title = " ".join(str(title or "").split())
+    if len(clean_title) > MAX_WIDGET_TITLE_CHARS:
+        raise WidgetError(f"title is longer than {MAX_WIDGET_TITLE_CHARS} characters")
+    if clean_title:
+        attributes.append(f'title="{_escape_attribute(clean_title)}"')
+    clean_id = str(widget_id or "").strip()
+    if clean_id:
+        if not _WIDGET_ID_RE.fullmatch(clean_id):
+            raise WidgetError("widget_id must be 1-64 characters of letters, digits, _ . : -")
+        attributes.append(f'id="{clean_id}"')
+    parsed_height = _height(height)
+    if parsed_height is not None:
+        attributes.append(f'height="{parsed_height}"')
+    opener = "<iframe" + "".join(f" {attribute}" for attribute in attributes) + ">"
+    block = f"{opener}\n{body}\n</iframe>"
+    lead = str(text or "").strip()
+    markdown = f"{lead}\n\n{block}" if lead else block
+    scan = scan_widgets(markdown)
+    if len(scan.widgets) != 1 or scan.unclosed_at is not None or scan.widgets[0].end != len(markdown):
+        raise WidgetError("text must not contain another <iframe> or an unclosed code fence; a message holds one widget")
+    if utf16_len(markdown) > MAX_MESSAGE_TEXT_UTF16_UNITS or len(markdown.encode("utf-8")) > MAX_MESSAGE_TEXT_UTF8_BYTES:
+        raise WidgetError("message exceeds Inline's 100,000 character limit")
+    return markdown