@mseep/open-computer-use 1.0.0

package/openwebui/patches/fix_large_tool_results.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: FSL-1.1-Apache-2.0
+# Copyright (c) 2025 Open Computer Use Contributors
+"""Patch for Open WebUI: truncate large tool results in output (DB + LLM).
+
+Problem: MCP tools (Metabase, OpenSearch, Playwright, etc.) return
+results 15K-100K+ chars. After 2-3 calls the model context is exhausted.
+Full results are also saved to DB, bloating chat storage.
+
+Solution: truncate results in output in-place BEFORE they reach
+serialize_content_blocks (-> DB) and convert_content_blocks_to_messages (-> LLM).
+Single interception point: in tool loop before _saved_output deep copy.
+
+If ORCHESTRATOR_URL is set, full result is uploaded as a file
+to computer-use-server, model receives preview + file_path.
+Otherwise, only preview + truncation notice.
+
+Two intercept points:
+  Mod 2: Tool loop -- truncate CURRENT tool results before LLM + DB
+  Mod 3: History -- truncate OLD tool results loaded from DB before LLM
+
+Config env vars:
+  TOOL_RESULT_MAX_CHARS (default: 50000) -- truncation threshold (~12.5K tokens). 0 = disable
+  TOOL_RESULT_PREVIEW_CHARS (default: 2000) -- preview size
+  ORCHESTRATOR_URL -- internal URL of computer-use-server for large-result uploads
+
+Must run AFTER fix_tool_loop_errors.py (Mod 2 targets its marker).
+Target: Open WebUI 0.9.5
+"""
+
+import os
+import sys
+
+_PATCH_TARGET_OVERRIDE = os.environ.get("_PATCH_TARGET_OVERRIDE", "")
+MIDDLEWARE_PATH = _PATCH_TARGET_OVERRIDE or "/app/backend/open_webui/utils/middleware.py"
+
+PATCH_MARKER = "_truncate_large_results_in_output"
+NEW_PATCH_MARKER = "FIX_LARGE_TOOL_RESULTS"
+
+FUNCTION_CODE = '''
+# === PATCH: _truncate_large_results_in_output -- truncate large MCP tool results (DB + LLM) ===
+# FIX_LARGE_TOOL_RESULTS
+import os as _os_module
+
+_TOOL_RESULT_MAX_CHARS = int(_os_module.environ.get('TOOL_RESULT_MAX_CHARS', '50000'))
+_TOOL_RESULT_PREVIEW_CHARS = int(_os_module.environ.get('TOOL_RESULT_PREVIEW_CHARS', '2000'))
+_ORCHESTRATOR_URL = _os_module.environ.get('ORCHESTRATOR_URL', '').rstrip('/')
+
+
+async def _upload_result_to_docker_ai(content: str, filename: str, chat_id: str) -> str:
+    """Upload full tool result to docker-ai container. Returns file_path or empty string."""
+    if not _ORCHESTRATOR_URL or not chat_id:
+        return ''
+    try:
+        import aiohttp
+        upload_url = f'{_ORCHESTRATOR_URL}/api/uploads/{chat_id}/{filename}'
+        form = aiohttp.FormData()
+        form.add_field('file', content.encode('utf-8'), filename=filename, content_type='text/plain')
+        async with aiohttp.ClientSession() as session:
+            async with session.post(upload_url, data=form, timeout=aiohttp.ClientTimeout(total=15)) as resp:
+                if resp.status < 300:
+                    return f'/mnt/user-data/uploads/{filename}'
+                log.warning('TOOL_RESULT_UPLOAD: status=%d url=%s', resp.status, upload_url)
+    except Exception as e:
+        log.warning('TOOL_RESULT_UPLOAD_ERR: %s', e)
+    return ''
+
+
+async def _truncate_large_results_in_output(output: list, chat_id: str) -> None:
+    """Truncate large tool results in output blocks IN-PLACE.
+
+    Iterates output blocks, finds tool_calls results with content > threshold,
+    optionally uploads full content to docker-ai, replaces with preview.
+    Affects both DB storage (serialize_content_blocks) and LLM context
+    (convert_content_blocks_to_messages) since both read from the same output.
+    """
+    if _TOOL_RESULT_MAX_CHARS <= 0:
+        return
+    import time as _time
+    _trunc_msg = lambda size_kb, preview, fpath: (
+        (f'[Tool result truncated: {size_kb:.0f} KB]\\n'
+         f'Preview (first {_TOOL_RESULT_PREVIEW_CHARS} chars):\\n'
+         f'{preview}\\n\\n---\\n'
+         f'Full result saved to: {fpath}\\n'
+         f'Read with: view {fpath} view_range=[1, 50]')
+        if fpath else
+        (f'[Tool result truncated: {size_kb:.0f} KB -> first {_TOOL_RESULT_PREVIEW_CHARS} chars]\\n'
+         f'{preview}\\n\\n---\\n'
+         f'[Result truncated. Full output was {size_kb:.0f} KB.]')
+    )
+    for block in output:
+        btype = block.get('type', '')
+        # --- Chat Completions format: type=tool_calls, results[].content ---
+        if btype == 'tool_calls':
+            tool_names = {}
+            for tc in block.get('content', []):
+                if not isinstance(tc, dict):
+                    continue
+                tc_id = tc.get('id', '')
+                tc_name = tc.get('function', {}).get('name', '') if isinstance(tc.get('function'), dict) else ''
+                if tc_id and tc_name:
+                    tool_names[tc_id] = tc_name
+            for result in block.get('results', []):
+                content = result.get('content', '')
+                if not isinstance(content, str) or len(content) <= _TOOL_RESULT_MAX_CHARS:
+                    continue
+                size_kb = len(content) / 1024
+                tc_id = result.get('tool_call_id', 'unknown')
+                tool_name = tool_names.get(tc_id, 'tool')
+                preview = content[:_TOOL_RESULT_PREVIEW_CHARS]
+                file_path = ''
+                if _ORCHESTRATOR_URL and chat_id:
+                    fname = f'tool_result_{tool_name}_{tc_id[:8]}_{int(_time.time())}.txt'
+                    file_path = await _upload_result_to_docker_ai(content, fname, chat_id)
+                result['content'] = _trunc_msg(size_kb, preview, file_path)
+                log.info('TOOL_RESULT_TRUNCATED: fmt=tool_calls tool=%s orig_kb=%.1f uploaded=%s',
+                         tool_name, size_kb, bool(file_path))
+        # --- Responses API format: type=function_call_output, output[].text ---
+        elif btype == 'function_call_output':
+            for part in block.get('output', []):
+                if part.get('type') != 'input_text':
+                    continue
+                text = part.get('text', '')
+                if not isinstance(text, str) or len(text) <= _TOOL_RESULT_MAX_CHARS:
+                    continue
+                size_kb = len(text) / 1024
+                preview = text[:_TOOL_RESULT_PREVIEW_CHARS]
+                call_id = block.get('call_id', 'unknown')
+                file_path = ''
+                if _ORCHESTRATOR_URL and chat_id:
+                    fname = f'tool_result_{call_id[:12]}_{int(_time.time())}.txt'
+                    file_path = await _upload_result_to_docker_ai(text, fname, chat_id)
+                part['text'] = _trunc_msg(size_kb, preview, file_path)
+                log.info('TOOL_RESULT_TRUNCATED: fmt=function_call_output call_id=%s orig_kb=%.1f uploaded=%s',
+                         call_id[:12], size_kb, bool(file_path))
+
+
+def _truncate_tool_messages_in_history(messages: list) -> None:
+    """Truncate large tool result messages from chat history IN-PLACE.
+
+    Sync version (no upload) for old results loaded from DB via
+    process_messages_with_output(). Prevents context window overflow
+    from accumulated tool results in long chat sessions.
+    """
+    if _TOOL_RESULT_MAX_CHARS <= 0:
+        return
+    for msg in messages:
+        if msg.get('role') != 'tool':
+            continue
+        content = msg.get('content', '')
+        if not isinstance(content, str) or len(content) <= _TOOL_RESULT_MAX_CHARS:
+            continue
+        size_kb = len(content) / 1024
+        preview = content[:_TOOL_RESULT_PREVIEW_CHARS]
+        msg['content'] = (
+            f'[Tool result from history truncated: {size_kb:.0f} KB -> first {_TOOL_RESULT_PREVIEW_CHARS} chars]\\n'
+            f'{preview}\\n\\n---\\n'
+            f'[Result truncated. Full output was {size_kb:.0f} KB.]'
+        )
+        log.info('TOOL_RESULT_HISTORY_TRUNCATED: orig_kb=%.1f', size_kb)
+# === END PATCH: _truncate_large_results_in_output ===
+'''
+
+# Search pattern: targets code AFTER fix_tool_loop_errors.py (TOOL_LOOP_ERRORS_UNIFIED marker).
+# v0.9.2: Patch 3's REPLACE now emits `'metadata': metadata,` inside new_form_data — include it
+# in the SEARCH so the cascade region is fully covered and any upstream drift of the metadata
+# key fails loud here rather than silently producing a broken middleware.
+SEARCH_TOOL_LOOP = (
+    "                    _saved_output = json.loads(json.dumps(output))"
+    "  # TOOL_LOOP_ERRORS_UNIFIED: save for restore on error\n"
+    "                    try:\n"
+    "                        new_form_data = {\n"
+    "                            **form_data,\n"
+    "                            'model': model_id,\n"
+    "                            'stream': True,\n"
+    "                            'metadata': metadata,\n"
+    "                        }\n"
+)
+
+REPLACE_TOOL_LOOP = (
+    "                    await _truncate_large_results_in_output("
+    "output, metadata.get('chat_id', ''))  # LARGE_TOOL_RESULTS\n"
+    "                    _saved_output = json.loads(json.dumps(output))"
+    "  # TOOL_LOOP_ERRORS_UNIFIED: save for restore on error\n"
+    "                    try:\n"
+    "                        new_form_data = {\n"
+    "                            **form_data,\n"
+    "                            'model': model_id,\n"
+    "                            'stream': True,\n"
+    "                            'metadata': metadata,\n"
+    "                        }\n"
+)
+
+
+# Mod 3: Truncate historical tool messages loaded from DB
+SEARCH_HISTORY = (
+    "    form_data['messages'] = process_messages_with_output(\n"
+    "        form_data.get('messages', []),\n"
+    "        reasoning_format=get_reasoning_format(model),\n"
+    "    )\n"
+    "\n"
+    "    system_message = get_system_message(form_data.get('messages', []))\n"
+)
+
+REPLACE_HISTORY = (
+    "    form_data['messages'] = process_messages_with_output(\n"
+    "        form_data.get('messages', []),\n"
+    "        reasoning_format=get_reasoning_format(model),\n"
+    "    )\n"
+    "    _truncate_tool_messages_in_history(form_data['messages'])  # LARGE_TOOL_RESULTS: trim old results\n"
+    "\n"
+    "    system_message = get_system_message(form_data.get('messages', []))\n"
+)
+
+
+def apply_patch():
+    """Apply large tool results truncation patch to middleware.py."""
+
+    if not os.path.exists(MIDDLEWARE_PATH):
+        print(
+            f"ERROR: fix_large_tool_results target file {MIDDLEWARE_PATH} not found. "
+            "Refusing to produce a silently-broken image.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    with open(MIDDLEWARE_PATH, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Check if already applied
+    if PATCH_MARKER in content or NEW_PATCH_MARKER in content:
+        print(f"ALREADY PATCHED: {MIDDLEWARE_PATH} contains {PATCH_MARKER}")
+        return True
+
+    # Mod 1: Inject functions after imports
+    import_marker = "from open_webui.models.chats import Chats"
+    marker_idx = content.find(import_marker)
+    if marker_idx < 0:
+        print(
+            f"ERROR: fix_large_tool_results Mod 1 anchor (import marker "
+            f"'from open_webui.models.chats import Chats') not found in {MIDDLEWARE_PATH} "
+            "— upstream may have restructured imports. "
+            "Refusing to produce a silently-broken image.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    eol_idx = content.index("\n", marker_idx) + 1
+    content = content[:eol_idx] + FUNCTION_CODE + content[eol_idx:]
+    print("  [1/3] Injected functions: _truncate_large_results_in_output, _truncate_tool_messages_in_history, _upload_result_to_docker_ai")
+
+    # Mod 2: Add truncation call before _saved_output in tool loop
+    if SEARCH_TOOL_LOOP not in content:
+        print(
+            "ERROR: fix_large_tool_results Mod 2 anchor (TOOL_LOOP_ERRORS_UNIFIED marker) "
+            "not found. Run fix_tool_loop_errors.py first, or upstream tool-loop structure "
+            "changed. Refusing to produce a silently-broken image.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    content = content.replace(SEARCH_TOOL_LOOP, REPLACE_TOOL_LOOP, 1)
+    print("  [2/3] Tool loop: truncate current results before _saved_output")
+
+    # Mod 3: Truncate historical tool messages from DB
+    if SEARCH_HISTORY not in content:
+        print(
+            "ERROR: fix_large_tool_results Mod 3 anchor (process_messages_with_output / "
+            "get_system_message block) not found in middleware.py — upstream may have "
+            "restructured. Refusing to produce a silently-broken image.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    content = content.replace(SEARCH_HISTORY, REPLACE_HISTORY, 1)
+    print("  [3/3] History: truncate old tool results from DB")
+
+    with open(MIDDLEWARE_PATH, "w", encoding="utf-8") as f:
+        f.write(content)
+
+    print("PATCHED: fix_large_tool_results applied successfully.")
+    print("  Config: TOOL_RESULT_MAX_CHARS (default 50000), TOOL_RESULT_PREVIEW_CHARS (default 2000)")
+    print("  Upload: ORCHESTRATOR_URL (optional, for Computer Use)")
+    print("  Log markers: TOOL_RESULT_TRUNCATED, TOOL_RESULT_HISTORY_TRUNCATED")
+    return True
+
+
+if __name__ == "__main__":
+    print("Applying large tool results truncation patch to middleware.py...")
+    success = apply_patch()
+    sys.exit(0 if success else 1)

package/openwebui/patches/fix_preview_url_detection.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: FSL-1.1-Apache-2.0
+# Copyright (c) 2025 Open Computer Use Contributors
+"""
+Patch for Open WebUI 0.9.5 (host-agnostic): automatic detection of file URLs in messages
+
+Problem: To show file preview in the Artifacts panel, we need to detect
+Computer Use Server file links in assistant messages and auto-open the
+preview SPA in the Artifacts panel.
+
+Solution: Patch getCodeBlockContents() -- the function that parses message content.
+Push an iframe into htmlGroups array (e): if no code blocks found but content has
+a link to /files/{chat_id}/... or /preview/{chat_id} -- push iframe artifact.
+The existing chain (getContents -> artifactContents -> auto-show) handles the rest.
+
+Host-agnostic: the iframe src is reconstructed at runtime from the matched URL's own
+origin, so no build-time host configuration is needed.
+
+=== Compiled-code anchor ===
+
+  Wn=r=>{r=Ce(r);const t=r.match(/```[\\s\\S]*?```/g);let n=[],e=[];
+  ... // code block parsing fills e[] with {html, css, js} groups
+  const i=e.map(o=>o.html).join(""),a=e.map(o=>o.css).join(""),l=e.map(o=>o.js).join("");
+  return{codeBlocks:n,html:i.trim(),css:a.trim(),js:l.trim(),htmlGroups:e.filter(...).map(...)}
+
+Injection point: BEFORE `const i=e.map(o=>o.html).join("")`
+We push our iframe artifact into `e` (mutable let array), then `const i` naturally
+picks it up. No const reassignment needed.
+
+  // INJECTED:
+  /* FIX_PREVIEW_URL_DETECTION */
+  if(!e.some(o=>o.html)&&r&&/\\/(files|preview)\\//.test(r)){
+    var _pm=r.match(/regex/); if(_pm) e.push({html:'<iframe...>',css:'...',js:''});
+  }
+  const i=e.map(o=>o.html).join(""); // now includes our iframe
+
+Fail-loud contract: if the anchor regex does not match any chunk, the script
+exits with sys.exit(1) and prints "ERROR: ..." on stderr. This intentionally
+fails the Docker build rather than producing a silently-broken image.
+"""
+
+import os
+import sys
+import glob
+import re
+
+BUILD_CHUNKS_DIR = "/app/build/_app/immutable/chunks"
+
+# Search pattern: the const declarations right before return
+# We match: })}const i=e.map(o=>o.html).join("")
+CONST_DECL_PATTERN = re.compile(
+    r'(\}\)\})const (\w+)=(\w+)\.map\(\w+=>\w+\.html\)\.join\(""\)'
+)
+
+# Context marker: the backtick regex (confirms we're in getCodeBlockContents)
+BACKTICK_REGEX_MARKER = r'[\s\S]*?'  # part of /```[\s\S]*?```/g
+
+# This patch is host-agnostic. The iframe `src` is reconstructed at runtime
+# from the matched URL's own origin (_pm[1] in the injected JS), so no
+# build-time host configuration is consumed here. COMPUTER_USE_SERVER_URL
+# is intentionally not read by this patch.
+
+# Idempotency marker -- injected as a JS comment at the patch site.
+# Presence of this marker indicates the chunk has already been patched by this script.
+IDEMPOTENCY_MARKER = "/* FIX_PREVIEW_URL_DETECTION */"
+# Legacy marker -- continue to recognise chunks patched by v0.8.12 runs.
+LEGACY_PATCHED_MARKER = "preview-url-detect"
+
+
+def find_chunk_file():
+    """Find JS chunk containing getCodeBlockContents (by const pattern + backtick regex)"""
+    if not os.path.isdir(BUILD_CHUNKS_DIR):
+        print(f"ERROR: Directory not found: {BUILD_CHUNKS_DIR}", file=sys.stderr)
+        return None
+
+    for filepath in sorted(glob.glob(os.path.join(BUILD_CHUNKS_DIR, "*.js"))):
+        if filepath.endswith(".map"):
+            continue
+        try:
+            with open(filepath, "r", encoding="utf-8") as f:
+                content = f.read()
+            # Check for already-patched file first (new or legacy marker)
+            if (IDEMPOTENCY_MARKER in content) or (LEGACY_PATCHED_MARKER in content and '_pm' in content):
+                if BACKTICK_REGEX_MARKER in content:
+                    return filepath
+            # Check for unpatched file
+            has_const = CONST_DECL_PATTERN.search(content)
+            has_backtick = BACKTICK_REGEX_MARKER in content
+            if has_const and has_backtick:
+                return filepath
+        except Exception as e:
+            print(f"  Warning: Could not read {filepath}: {e}")
+            continue
+
+    return None
+
+
+def extract_param_name(content, const_match):
+    """Extract the function parameter name (the 'content' parameter).
+
+    Looks backwards from the const declaration to find PARAM.match(/```...)
+    """
+    before = content[max(0, const_match.start() - 2000):const_match.start()]
+    param_match = re.search(r'(\w+)\.match\(/```', before)
+    if param_match:
+        return param_match.group(1)
+    return None
+
+
+def build_injection(param_name, groups_var):
+    """Build the URL detection code to inject before const declarations.
+
+    Pushes an iframe artifact into the htmlGroups array (e).
+    The array is mutable (let), so push works in strict mode.
+
+    Host-agnostic: matches any HTTP(S) URL whose path contains /files/<id>/
+    or /preview/<id>. The actual host is read back from the matched URL
+    itself, so the filter's X-Public-Base-URL (localhost, 127.0.0.1,
+    computer-use-server, public domain, etc.) all work without the patch
+    having to know what it is at build time.
+    """
+    # Build the JS expression that, at runtime, produces an iframe tag string.
+    # Two literal fragments + two interpolations (_pm[1]=origin, _pm[2]=id).
+    iframe_html_expr = (
+        "'<iframe src=\"'+_pm[1]+'/preview/'+_pm[2]+"
+        "'\" style=\"width:100%;height:100%;border:none\" "
+        "allow=\"clipboard-write; keyboard-map\"></iframe>\\n'"
+    )
+    iframe_css = (
+        "'*{margin:0;padding:0;overflow:hidden}"
+        "html,body{height:100%}\\n'"
+    )
+    # Host-agnostic detector: /(files|preview)\/ in url path, then capture
+    # the origin (_pm[1]) and the id segment (_pm[2]) via a single regex.
+    # Guard with `!param.some(o=>o.html)` so genuine fenced <html> blocks
+    # still take precedence.
+    # NOTE: we match origin http(s)://host[:port] then /files|/preview/ then id.
+    return (
+        f'{IDEMPOTENCY_MARKER}'
+        f'if(!{groups_var}.some(o=>o.html)&&{param_name}&&'
+        f'/\\/(files|preview)\\//.test({param_name}))'
+        f'{{var _pm={param_name}.match('
+        f'/(https?:\\/\\/[^\\/\\s\\"\\)]+)\\/(?:files|preview)\\/([^\\/\\s\\"\\)]+)/'
+        f');if(_pm){{{groups_var}.push({{html:{iframe_html_expr},css:{iframe_css},js:""}})}}}}'
+        f'/*{LEGACY_PATCHED_MARKER}*/'
+    )
+
+
+def apply_patch():
+    """Apply the preview URL detection patch to getCodeBlockContents."""
+    chunk_file = find_chunk_file()
+    if not chunk_file:
+        print("ERROR: Could not find JS chunk with getCodeBlockContents", file=sys.stderr)
+        print(f"  Searched in: {BUILD_CHUNKS_DIR}/*.js", file=sys.stderr)
+        return False
+
+    print(f"  Found chunk: {os.path.basename(chunk_file)}")
+
+    with open(chunk_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # --- Idempotency short-circuit ---
+    # Either the new-style marker or the legacy marker indicates a prior run
+    # has already patched this chunk. Declare success without mutating.
+    if IDEMPOTENCY_MARKER in content:
+        print(f"ALREADY PATCHED: {os.path.basename(chunk_file)} contains {IDEMPOTENCY_MARKER}")
+        return True
+    if LEGACY_PATCHED_MARKER in content and '_pm' in content:
+        print(f"ALREADY PATCHED: {os.path.basename(chunk_file)} contains legacy marker '{LEGACY_PATCHED_MARKER}'")
+        return True
+
+    # Find the const declaration: })}const i=e.map(o=>o.html).join("")
+    const_match = CONST_DECL_PATTERN.search(content)
+    if not const_match:
+        print("ERROR: Could not find const html declaration pattern", file=sys.stderr)
+        return False
+
+    # Extract variable names
+    prefix = const_match.group(1)  # })}
+    html_var = const_match.group(2)  # i
+    groups_var = const_match.group(3)  # e
+
+    print(f"  Variables: html={html_var}, htmlGroups={groups_var}")
+
+    # Extract parameter name
+    param_name = extract_param_name(content, const_match)
+    if not param_name:
+        print("ERROR: Could not determine content parameter name", file=sys.stderr)
+        return False
+
+    print(f"  Content param: {param_name}")
+
+    # Build injection code
+    injection = build_injection(param_name, groups_var)
+
+    print(f"  Injection ({len(injection)} chars):")
+    print(f"    {injection[:140]}...")
+
+    # Injection point: between })} and const i=...
+    # Replace: })}const i=...  ->  })}INJECTION;const i=...
+    old = const_match.group(0)
+    new = prefix + injection + "const " + old[len(prefix) + len("const "):]
+
+    content_new = content.replace(old, new, 1)
+
+    if content_new == content:
+        print("ERROR: Replacement had no effect", file=sys.stderr)
+        return False
+
+    with open(chunk_file, "w", encoding="utf-8") as f:
+        f.write(content_new)
+
+    print(f"PATCHED! File: {os.path.basename(chunk_file)}")
+    return True
+
+
+if __name__ == "__main__":
+    print("Applying Preview URL detection patch to Open WebUI frontend...")
+    success = apply_patch()
+    if not success:
+        print(
+            "ERROR: fix_preview_url_detection anchor not found in "
+            f"{BUILD_CHUNKS_DIR}/*.js -- getCodeBlockContents compiled shape changed. "
+            "Check v0.9.5 source + update CONST_DECL_PATTERN. "
+            "Refusing to produce a silently-broken image.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    print("PATCHED: fix_preview_url_detection applied successfully.")
+    sys.exit(0)