scout-ai 1.2.3 → 2.0.0

data/lib/scout-ai.rb

@@ -7,6 +7,7 @@ Path.add_path :scout_ai_lib, File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL
 require 'scout/llm/ask'
 require 'scout/llm/chat'
 require 'scout/llm/embed'
+require 'scout/llm/image'
 require 'scout/llm/agent'
 
 

data/python/README.md

@@ -11,6 +11,11 @@ The package provides a lightweight Python interface to Scout-AI chats and agents
 
 In practice, the Python layer builds chat objects, serializes them through the Scout-AI CLI, and delegates execution back to Scout.
 
+Two complementary Python stories exist in this repository:
+
+- `python/README.md` — use Scout-AI chats and agents from Python
+- `doc/PythonAgentTasks.md` — write Python-backed workflow tasks that Ruby agents can load through `PythonWorkflow`
+
 ## Requirements
 
 The Python package expects the `scout-ai` command to be available.
@@ -45,7 +50,43 @@ For Hugging Face / RLHF helpers:
 
     pip install "scout-ai[huggingface] @ git+https://github.com/mikisvaz/scout-ai.git@main#subdirectory=python"
 
-## Quick example
+## Design in one paragraph
+
+The Python package is intentionally thin.
+
+- `Chat` builds an in-memory list of Scout chat messages.
+- `Agent` wraps a Scout-AI agent with eager `start_chat` / `current_chat` semantics.
+- `ScoutRunner` calls Ruby CLI commands.
+- Serialization is delegated to `scout-ai llm json`.
+- Execution is delegated to `scout-ai llm ask` or `scout-ai agent ask`.
+
+That means Python does not reimplement Scout chat parsing or workflow/tool execution.
+
+## Main classes
+
+- `scout_ai.Chat` — message builder and plain LLM chat runner
+- `scout_ai.Agent` — thin wrapper over Scout agents with eager `current_chat`
+- `scout_ai.Message` — message wrapper returned by `chat()`
+- `scout_ai.ScoutRunner` — CLI bridge used internally
+- `scout_ai.load_agent(name, ...)` — convenience constructor
+
+## Quick `Chat` example
+
+    from scout_ai import Chat
+
+    chat = Chat()
+    chat.endpoint("nano")
+    chat.system("You are concise")
+    chat.user("Say hello")
+
+    delta = chat.ask()
+    print(delta.to_json())
+
+    last = chat.chat()
+    print(last.role)
+    print(last.content)
+
+## Quick `Agent` example
 
     from scout_ai import load_agent
 
@@ -56,16 +97,153 @@ For Hugging Face / RLHF helpers:
     message = agent.chat()
     print(message.content)
 
-## Chat example
+A more tool-oriented example:
+
+    from scout_ai import load_agent
+
+    agent = load_agent("Planner", endpoint="deep")
+    agent.import_("context.chat")
+    agent.file("README.md")
+    agent.tool("ComputerUse searxng url")
+    agent.user("Summarize the file and search for a few references")
+
+    message = agent.chat()
+    print(message.content)
+
+## `ask()` vs `chat()` semantics
+
+This distinction is the most important part of the API.
+
+### `Chat.ask()`
+
+- serializes the current chat to a temporary file
+- runs `scout-ai llm ask -c <tmp_file>`
+- reloads the resulting chat file
+- computes the delta between the original and updated message lists
+- returns a new `Chat` object containing only the new messages
+- does not mutate the original `Chat`
+
+### `Chat.chat()`
+
+- performs the same execution as `ask()`
+- appends the new messages to the current `Chat`
+- returns the last meaningful new message
+- skips a trailing bookkeeping message such as `previous_response_id`
+
+### `Agent.ask()`
+
+- uses the agent-aware command `scout-ai agent ask <agent_name> -c <tmp_file>`
+- returns a delta `Chat`
+- does not mutate `current_chat`
+
+### `Agent.chat()`
+
+- uses the same agent-aware command
+- appends the new messages to `current_chat`
+- returns the last meaningful new message
+
+This mirrors how Scout-AI distinguishes between returning a trace/delta and advancing the active conversation.
+
+## Message builder methods
+
+`Chat` exposes a Python-side builder API that mirrors the Ruby chat builder closely.
+
+Common methods include:
+
+- `user(text)`
+- `system(text)`
+- `assistant(text)`
+- `file(path)`
+- `directory(path)`
+- `image(path)`
+- `pdf(path)`
+- `import_(path)`
+- `import_last(path)`
+- `continue_(path)`
+- `tool(spec)`
+- `use(spec)`
+- `introduce(workflow)`
+- `task(workflow, task_name, **inputs)`
+- `inline_task(...)`
+- `exec_task(...)`
+- `job(step_or_path)`
+- `inline_job(step_or_path)`
+- `association(name, path, **options)`
+- `endpoint(value)`
+- `model(value)`
+- `backend(value)`
+- `format(value)`
+- `option(name, value)`
+- `sticky_option(name, value)`
+- `persist(value=True)`
+- `previous_response_id(value)`
+
+Notes:
+
+- Python uses `import_()` and `continue_()` because `import` and `continue` are reserved keywords.
+- `use()` is just an alias for `tool()`.
+- endpoint and model settings are represented as messages, just like in Ruby Scout-AI chats.
+
+## Saving and loading chats
+
+You can save or load the Python chat wrapper in either Scout chat format or JSON format.
 
     from scout_ai import Chat
 
-    chat = Chat()
-    chat.system("You are concise")
-    chat.user("Say hello")
+    chat = Chat().system("You are helpful").user("Summarize this")
+    chat.save_chat("example.chat")
+    chat.save_json("example.json")
 
-    delta = chat.ask()
-    print(delta.to_json())
+    loaded_chat = Chat.load("example.chat", input_format="chat")
+    loaded_json = Chat.load("example.json", input_format="json")
+
+The rendered Scout chat text is also available directly:
+
+    text = chat.render()
+    print(text)
+
+## How the conversion works
+
+The Python package deliberately does not implement Scout chat parsing or printing itself.
+
+Instead it always round-trips through:
+
+    scout-ai llm json
+
+That command is used to:
+
+- convert JSON messages to Scout chat text
+- convert Scout chat text back to JSON messages
+
+This keeps Ruby as the authority for the chat format.
+
+## Eager agent initialization
+
+`load_agent(name, ...)` initializes `start_chat` and `current_chat` eagerly.
+
+In the current implementation, the Python wrapper tries to resolve the agent path through:
+
+    scout-ai agent find <agent_name>
+
+and then loads the agent's `start_chat` file when it can find one.
+
+That works best for agent directories with an explicit `start_chat` file. If an agent's initial state is synthesized indirectly on the Ruby side rather than stored as a file, the Python wrapper may start from a smaller initial chat.
+
+## Relationship to Python-backed workflow tasks
+
+The Python package described here is for using chats and agents from Python.
+
+It is separate from writing workflow tasks in Python.
+
+If you want to create Python functions that Ruby agents can load and use as workflow tasks, see:
+
+    ../doc/PythonAgentTasks.md
+
+That document explains:
+
+- `scout.task(...)`
+- `PythonWorkflow`
+- agent directory auto-loading from `python/*.py`
 
 ## What pip installs
 
@@ -77,15 +255,20 @@ That separation is intentional:
 - Ruby remains responsible for the actual Scout-AI runtime
 - Python provides an ergonomic interface to that runtime
 
-## Package layout
-
-- `scout_ai.chat.Chat` — chat builder and execution wrapper
-- `scout_ai.agent.Agent` — thin wrapper over Scout agents with eager `current_chat`
-- `scout_ai.runner.ScoutRunner` — CLI bridge used internally
-- `scout_ai.message.Message` — message wrapper returned by `chat()`
-
 ## Running tests
 
 From the repository root:
 
     PYTHONPATH=python python -m unittest discover python/tests
+
+A minimal smoke test can also round-trip a chat through the real CLI bridge:
+
+    PYTHONPATH=python python - <<'PY'
+    from scout_ai import Chat, ScoutRunner
+
+    runner = ScoutRunner(command=['bin/scout-ai'])
+    chat = Chat(runner=runner).system('You are concise').user('Hello world')
+    text = chat.render()
+    print(text)
+    print(Chat.from_text(text, runner=runner).to_json())
+    PY

data/python/scout_ai/huggingface/eval.py

@@ -1,10 +1,16 @@
+import json
+import re
+
+
 def forward(model, features):
     return model(**features)
 
+
 def get_logits(predictions):
     logits = predictions["logits"]
     return [v.detach().cpu().numpy() for v in logits]
 
+
 def eval_model(model, tokenizer, texts, return_logits=True):
     features = tokenizer(texts, return_tensors='pt', truncation=True).to(model.device)
     model.eval()
@@ -13,48 +19,253 @@ def eval_model(model, tokenizer, texts, return_logits=True):
         return get_logits(predictions)
     return predictions
 
-def eval_causal_lm_chat(
+
+def _move_to_device(inputs, device):
+    if hasattr(inputs, "to"):
+        return inputs.to(device)
+
+    if isinstance(inputs, dict):
+        return {
+            key: value.to(device) if hasattr(value, "to") else value
+            for key, value in inputs.items()
+        }
+
+    return inputs
+
+
+def _prepare_tools(tools, tool_argument="tools"):
+    if tools is None:
+        return None
+
+    prepared = []
+    for tool in tools:
+        if isinstance(tool, dict):
+            tool = dict(tool)
+            function = tool.get("function")
+
+            if tool_argument != "tools" and isinstance(function, dict):
+                prepared.append(function)
+            elif tool_argument == "tools" and function is None and "name" in tool and "parameters" in tool:
+                prepared.append({"type": "function", "function": tool})
+            else:
+                prepared.append(tool)
+        else:
+            prepared.append(tool)
+
+    return prepared
+
+
+def _prepare_chat_inputs(
     model, tokenizer, messages,
+    tools=None,
     chat_template=None,
     chat_template_kwargs=None,
-    generation_kwargs=None
+    tool_argument=None,
 ):
-    """
-    Evaluate a CausalLM model given chat messages. Uses tokenizer's chat template by default.
-
-    Args:
-        model: Huggingface CausalLM
-        tokenizer: Huggingface tokenizer
-        messages: List[Dict[str, str]] (OpenAI API style, 'role' and 'content')
-        chat_template: (Optional) Override string for the chat template.
-        chat_template_kwargs: (Optional) Dict, kwargs for apply_chat_template (like tokenize, add_generation_prompt, etc).
-        generation_kwargs: (Optional) Dict for model.generate
-
-    Returns:
-        Generated text (or list, depending on settings).
-    """
-    chat_template_kwargs = chat_template_kwargs or {}
-    generation_kwargs = generation_kwargs or {}
-
-    # If the tokenizer has a chat template (HF 4.34+)
-    if hasattr(tokenizer, "___apply_chat_template"):
-        kwargs = dict(add_generation_prompt=True, tokenize=False)
+    chat_template_kwargs = dict(chat_template_kwargs or {})
+    tool_argument = tool_argument or "tools"
+
+    if hasattr(tokenizer, "apply_chat_template"):
+        kwargs = dict(
+            add_generation_prompt=True,
+            tokenize=True,
+            return_dict=True,
+            return_tensors="pt",
+        )
         kwargs.update(chat_template_kwargs)
+
         if chat_template is not None:
-            # Override the template (may require tokenizer._chat_template)
-            tokenizer._chat_template = chat_template
-        prompt = tokenizer.apply_chat_template(messages, **kwargs)
+            kwargs["chat_template"] = chat_template
+
+        prepared_tools = _prepare_tools(tools, tool_argument)
+        if prepared_tools:
+            kwargs[tool_argument] = prepared_tools
+
+        rendered = tokenizer.apply_chat_template(messages, **kwargs)
+
+        if isinstance(rendered, str):
+            inputs = tokenizer(rendered, return_tensors="pt")
+        else:
+            inputs = rendered
     else:
-        # Fallback: simple concatenation
-        prompt = "\n".join([msg['content'] for msg in messages])
+        prompt = "\n".join(str(message.get("content", "")) for message in messages)
+        inputs = tokenizer(prompt, return_tensors="pt")
+
+    return _move_to_device(inputs, model.device)
+
+
+def _decode_generated_text(tokenizer, inputs, output_ids):
+    input_ids = inputs["input_ids"]
+    return tokenizer.decode(
+        output_ids[0, input_ids.shape[1]:],
+        skip_special_tokens=True,
+    )
+
+
+def _normalize_tool_call(tool_call, index=0):
+    if tool_call is None:
+        return None
+
+    if "function" in tool_call and isinstance(tool_call["function"], dict):
+        function = dict(tool_call["function"])
+    else:
+        function = {
+            "name": tool_call.get("name"),
+            "arguments": tool_call.get("arguments", tool_call.get("parameters", {})),
+        }
+
+    arguments = function.get("arguments", {})
+    if isinstance(arguments, str):
+        try:
+            arguments = json.loads(arguments)
+        except Exception:
+            pass
+
+    return {
+        "id": tool_call.get("id") or tool_call.get("call_id") or f"call_{index}",
+        "type": "function",
+        "function": {
+            "name": function.get("name"),
+            "arguments": arguments,
+        },
+    }
+
+
+def _normalize_response(parsed, raw_text=None):
+    if parsed is None:
+        return {"role": "assistant", "content": (raw_text or "").strip()}
+
+    if isinstance(parsed, str):
+        return {"role": "assistant", "content": parsed.strip()}
+
+    if isinstance(parsed, list):
+        return {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [
+                tool_call
+                for tool_call in (
+                    _normalize_tool_call(tool_call, index)
+                    for index, tool_call in enumerate(parsed)
+                )
+                if tool_call is not None
+            ],
+        }
+
+    message = dict(parsed)
+    message.setdefault("role", "assistant")
+
+    if "tool_calls" in message:
+        message["tool_calls"] = [
+            tool_call
+            for tool_call in (
+                _normalize_tool_call(tool_call, index)
+                for index, tool_call in enumerate(message.get("tool_calls", []))
+            )
+            if tool_call is not None
+        ]
+    elif "name" in message and ("arguments" in message or "parameters" in message):
+        message["tool_calls"] = [_normalize_tool_call(message, 0)]
+        message.setdefault("content", "")
+
+    if message.get("content") is None:
+        message["content"] = ""
+
+    if raw_text is not None and "content" not in message:
+        message["content"] = raw_text.strip()
+
+    return message
+
+
+def _parse_tool_call_blocks(output_text):
+    matches = re.findall(r"<tool_call>\s*(.*?)\s*</tool_call>", output_text, re.DOTALL)
+    if not matches:
+        return None
+
+    tool_calls = []
+    for match in matches:
+        try:
+            payload = json.loads(match)
+        except Exception:
+            continue
+
+        payloads = payload if isinstance(payload, list) else [payload]
+        for item in payloads:
+            normalized = _normalize_tool_call(item, len(tool_calls))
+            if normalized is not None:
+                tool_calls.append(normalized)
+
+    if not tool_calls:
+        return None
+
+    content = re.sub(r"<tool_call>\s*.*?\s*</tool_call>", "", output_text, flags=re.DOTALL).strip()
+    return {
+        "role": "assistant",
+        "content": content,
+        "tool_calls": tool_calls,
+    }
+
+
+def parse_causal_lm_response(tokenizer, output_text, response_parser=None):
+    if response_parser not in (False, "false", "none"):
+        if hasattr(tokenizer, "parse_response"):
+            try:
+                parsed = tokenizer.parse_response(output_text)
+                normalized = _normalize_response(parsed, raw_text=output_text)
+                if normalized.get("tool_calls") or normalized.get("content") or normalized.get("thinking"):
+                    return normalized
+            except Exception:
+                pass
+
+        parsed = _parse_tool_call_blocks(output_text)
+        if parsed is not None:
+            return parsed
+
+    return {"role": "assistant", "content": output_text.strip()}
+
+
+def eval_causal_lm_chat(
+    model, tokenizer, messages,
+    chat_template=None,
+    chat_template_kwargs=None,
+    generation_kwargs=None,
+    tool_argument=None,
+):
+    generation_kwargs = dict(generation_kwargs or {})
+
+    inputs = _prepare_chat_inputs(
+        model, tokenizer, messages,
+        chat_template=chat_template,
+        chat_template_kwargs=chat_template_kwargs,
+        tool_argument=tool_argument,
+    )
 
-    # Tokenize as usual
-    inputs = tokenizer(prompt, return_tensors='pt').to(model.device)
     model.eval()
-    # Use generate
     output_ids = model.generate(**inputs, **generation_kwargs)
-    # Decode only the newly generated tokens (not the prompt)
-    output_text = tokenizer.decode(
-        output_ids[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True
+    return _decode_generated_text(tokenizer, inputs, output_ids)
+
+
+def eval_causal_lm_response(
+    model, tokenizer, messages,
+    tools=None,
+    chat_template=None,
+    chat_template_kwargs=None,
+    generation_kwargs=None,
+    tool_argument=None,
+    response_parser=None,
+):
+    generation_kwargs = dict(generation_kwargs or {})
+
+    chat_template = None
+    inputs = _prepare_chat_inputs(
+        model, tokenizer, messages,
+        tools=tools,
+        chat_template=chat_template,
+        chat_template_kwargs=chat_template_kwargs,
+        tool_argument=tool_argument,
     )
-    return output_text
+
+    model.eval()
+    output_ids = model.generate(**inputs, **generation_kwargs)
+    output_text = _decode_generated_text(tokenizer, inputs, output_ids)
+    return parse_causal_lm_response(tokenizer, output_text, response_parser=response_parser)

data/python/tests/test_huggingface_eval.py

@@ -0,0 +1,58 @@
+import unittest
+
+from scout_ai.huggingface.eval import parse_causal_lm_response
+
+
+class DummyTokenizer:
+    pass
+
+
+class ParserTokenizer:
+    def parse_response(self, text):
+        return {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "multiply",
+                        "arguments": {"a": 3, "b": 4},
+                    },
+                    "id": "call_1",
+                }
+            ],
+        }
+
+
+class HuggingfaceEvalTest(unittest.TestCase):
+    def test_parse_xml_tool_call_blocks(self):
+        tokenizer = DummyTokenizer()
+        text = '<tool_call>{"name": "multiply", "arguments": {"a": 3, "b": 4}}</tool_call>'
+
+        parsed = parse_causal_lm_response(tokenizer, text)
+
+        self.assertEqual(parsed["role"], "assistant")
+        self.assertEqual(parsed["tool_calls"][0]["function"]["name"], "multiply")
+        self.assertEqual(parsed["tool_calls"][0]["function"]["arguments"]["a"], 3)
+
+    def test_parse_tokenizer_response(self):
+        tokenizer = ParserTokenizer()
+        text = "ignored"
+
+        parsed = parse_causal_lm_response(tokenizer, text)
+
+        self.assertEqual(parsed["tool_calls"][0]["id"], "call_1")
+        self.assertEqual(parsed["tool_calls"][0]["function"]["arguments"]["b"], 4)
+
+    def test_parse_plain_text_when_disabled(self):
+        tokenizer = ParserTokenizer()
+        text = '<tool_call>{"name": "multiply", "arguments": {"a": 3, "b": 4}}</tool_call>'
+
+        parsed = parse_causal_lm_response(tokenizer, text, response_parser="none")
+
+        self.assertEqual(parsed, {"role": "assistant", "content": text})
+
+
+if __name__ == "__main__":
+    unittest.main()