octo-agent 0.11.2

data/lib/octo/message_history.rb

@@ -0,0 +1,373 @@
+# frozen_string_literal: true
+
+module Octo
+  # MessageHistory wraps the conversation message list and exposes
+  # business-meaningful operations instead of raw array manipulation.
+  #
+  # Internal fields (task_id, created_at, system_injected, etc.) are kept
+  # in the internal store but stripped when calling #to_api.
+  class MessageHistory
+    # Fields that are internal to the agent and must not be sent to the API.
+    INTERNAL_FIELDS = %i[
+      task_id created_at system_injected session_context memory_update
+      subagent_instructions subagent_result token_usage
+      compressed_summary chunk_path truncated transient
+      chunk_index chunk_count
+    ].freeze
+
+    def initialize(messages = [])
+      @messages = messages.dup
+    end
+
+    # ─────────────────────────────────────────────
+    # Write operations
+    # ─────────────────────────────────────────────
+
+    # Append a single message hash to the history.
+    #
+    # When appending a user message, automatically drop any trailing assistant
+    # message that has unanswered tool_calls (no tool_result follows it).
+    # This prevents API error 2013 ("tool call result does not follow tool call")
+    # when a previous task ended before observe() could append tool results
+    # (e.g. subagent crash, interrupt, or error).
+    def append(message)
+      if message[:role] == "user"
+        drop_dangling_tool_calls!
+      end
+      @messages << deep_sanitize_utf8(message)
+      self
+    end
+
+    # Replace (or insert at head) the system prompt message.
+    # Used by session_serializer#refresh_system_prompt.
+    def replace_system_prompt(content, **extra)
+      msg = { role: "system", content: content }.merge(extra)
+      idx = @messages.index { |m| m[:role] == "system" }
+      if idx
+        @messages[idx] = msg
+      else
+        @messages.unshift(msg)
+      end
+      self
+    end
+
+    # Replace the entire message list (used by compression rebuild).
+    def replace_all(new_messages)
+      @messages = new_messages.map { |m| deep_sanitize_utf8(m) }
+      self
+    end
+
+    # Remove and return the last message.
+    def pop_last
+      @messages.pop
+    end
+
+    # Remove all messages matching the block in-place.
+    # Generic history pruning utility — used by callers that need to
+    # strip transient/system-injected messages out of the persisted
+    # history (e.g. compaction, rollback on 400 errors).
+    def delete_where(&block)
+      @messages.reject!(&block)
+      self
+    end
+
+    # Mutate the last message matching the predicate lambda in-place.
+    # Used by execute_skill_with_subagent to update instruction messages.
+    def mutate_last_matching(predicate, &block)
+      msg = @messages.reverse.find { |m| predicate.call(m) }
+      block.call(msg) if msg
+      self
+    end
+
+    # Remove all messages from index onward (used by restore_session on error).
+    def truncate_from(index)
+      @messages = @messages[0...index]
+      self
+    end
+
+    # Roll back the history to just before the given message object.
+    # Removes the message and anything appended after it.
+    # Used to undo a failed speculative append (e.g. compression message that errored).
+    def rollback_before(message)
+      idx = @messages.index { |m| m.equal?(message) }
+      return self unless idx
+
+      @messages = @messages[0...idx]
+      self
+    end
+
+    # ─────────────────────────────────────────────
+    # Business queries
+    # ─────────────────────────────────────────────
+
+    # True when a system prompt message is present in the history.
+    # Used by inject_session_context to avoid injecting context messages
+    # before the system prompt has been built (which would cause the
+    # guard in run() to skip building it altogether).
+    def has_system_prompt?
+      @messages.any? { |m| m[:role] == "system" }
+    end
+
+    # True when the last assistant message has tool_calls but no
+    # tool_result has been appended yet (would cause a 400 from the API).
+    def pending_tool_calls?
+      return false if @messages.empty?
+
+      last = @messages.last
+      return false unless last[:role] == "assistant" && last[:tool_calls]&.any?
+
+      last_assistant_idx = @messages.rindex { |m| m == last }
+      @messages[(last_assistant_idx + 1)..].none? { |m| m[:role] == "tool" || m[:tool_results] }
+    end
+
+    # Return the session_date value from the most recent session_context message.
+    # Used by inject_session_context_if_needed to avoid re-injecting on the same date.
+    def last_session_context_date
+      msg = @messages.reverse.find { |m| m[:session_context] }
+      msg&.dig(:session_date)
+    end
+
+    # Return the chunk_count from the most recently injected chunk index message.
+    # Used by inject_chunk_index_if_needed to avoid re-injecting when nothing changed.
+    def last_injected_chunk_count
+      msg = @messages.reverse.find { |m| m[:chunk_index] }
+      msg&.dig(:chunk_count) || 0
+    end
+
+    # Return only real (non-system-injected) user messages.
+    def real_user_messages
+      @messages.select { |m| m[:role] == "user" && !m[:system_injected] }
+    end
+
+    # Return the index of the last real (non-system-injected) user message.
+    # Used by restore_session to trim back to a clean state on error.
+    def last_real_user_index
+      @messages.rindex { |m| m[:role] == "user" && !m[:system_injected] }
+    end
+
+    # Return the message with :subagent_instructions set.
+    def subagent_instruction_message
+      @messages.find { |m| m[:subagent_instructions] }
+    end
+
+    # Return all messages where task_id <= given id (Time Machine support).
+    def for_task(task_id)
+      @messages.select { |m| !m[:task_id] || m[:task_id] <= task_id }
+    end
+
+    # ─────────────────────────────────────────────
+    # Size helpers
+    # ─────────────────────────────────────────────
+
+    def size
+      @messages.size
+    end
+
+    def empty?
+      @messages.empty?
+    end
+
+    # Estimate total token count for all messages.
+    # Uses the ~4 chars/token heuristic (works well for English/code).
+    # Handles string content, array content blocks, and tool_calls.
+    def estimate_tokens
+      @messages.sum { |m| estimate_message_tokens(m) }
+    end
+
+    # ─────────────────────────────────────────────
+    # Output
+    # ─────────────────────────────────────────────
+
+    # Return a clean copy of messages suitable for sending to the LLM API:
+    # - strips internal-only fields
+    # - pads reasoning_content on synthetic assistant messages when the
+    #   conversation is running against a thinking-mode provider
+    #
+    # @param force_reasoning_content_pad [Boolean]
+    #   When true, unconditionally pad every assistant message that lacks a
+    #   reasoning_content field with an empty string. This is set by the
+    #   LLM caller AFTER a 400 "reasoning_content must be passed back" error
+    #   as a one-shot retry signal — the history-evidence heuristic below
+    #   can't fire when the previous turns came from a provider that keeps
+    #   thinking inline (e.g. MiniMax: <think>...</think> in content), so
+    #   this bypass lets us recover on the retry without a server restart.
+    def to_api(force_reasoning_content_pad: false)
+      msgs = @messages.map { |m| strip_for_api(m) }
+      ensure_reasoning_content_consistency(msgs, force: force_reasoning_content_pad)
+    end
+
+    # Return a shallow copy of the message list, excluding transient messages.
+    # Transient messages are valid during the current session but must not be
+    # persisted to session.json.
+    # For serialization, compression, and cloning.
+    def to_a
+      @messages.reject { |m| m[:transient] }.dup
+    end
+
+    # Estimate token count for a single message (role overhead + content).
+    private def estimate_message_tokens(message)
+      # ~4 tokens of overhead per message (role, formatting)
+      tokens = 4
+      tokens += estimate_content_tokens(message[:content])
+
+      # tool_calls: each call adds name + arguments chars
+      if message[:tool_calls].is_a?(Array)
+        message[:tool_calls].each do |tc|
+          tokens += estimate_content_tokens(tc.dig(:function, :name))
+          tokens += estimate_content_tokens(tc.dig(:function, :arguments))
+        end
+      end
+
+      tokens
+    end
+
+    # Estimate tokens from a content value (string, array of blocks, or nil).
+    # Heuristic: ASCII/code ~4 chars/token; CJK/multibyte ~1.5 chars/token.
+    private def estimate_content_tokens(content)
+      case content
+      when String
+        ascii_chars = content.scan(/[ -~]/).length
+        multibyte_chars = content.length - ascii_chars
+        ((ascii_chars / 4.0) + (multibyte_chars / 1.5)).ceil
+      when Array
+        content.sum do |block|
+          block.is_a?(Hash) ? estimate_content_tokens(block[:text] || block["text"]) : 0
+        end
+      else
+        0
+      end
+    end
+
+    # Drop the trailing assistant message if it has tool_calls with no subsequent
+    # tool_result — i.e. the tool call was never answered (dangling).
+    # Called automatically before appending any user message.
+    private def drop_dangling_tool_calls!
+      return unless pending_tool_calls?
+
+      @messages.pop
+    end
+
+    private def strip_for_api(message)
+      msg = strip_internal_fields(message)
+      content = msg[:content]
+      return msg unless content.is_a?(Array)
+
+      cleaned = content.filter_map do |block|
+        next block unless block.is_a?(Hash)
+
+        if block[:type] == "image_url" &&
+            block.dig(:image_url, :url) == "[image stripped]"
+          next nil
+        end
+
+        block.key?(:image_path) ? block.reject { |k, _| k == :image_path } : block
+      end
+
+      return msg if cleaned == content
+
+      if cleaned.empty?
+        msg.merge(content: "[images were shown to you in a previous turn]")
+      else
+        msg.merge(content: cleaned)
+      end
+    end
+
+    private def strip_internal_fields(message)
+      message.reject { |k, _| INTERNAL_FIELDS.include?(k) }
+    end
+
+    # Detect thinking-mode providers purely from history content and pad
+    # synthetic assistant messages with an empty reasoning_content when needed.
+    #
+    # WHY: Providers like DeepSeek V4 and Kimi K2 in thinking mode return a
+    # `reasoning_content` field on every assistant turn and REQUIRE the caller
+    # to echo a `reasoning_content` field back on every subsequent assistant
+    # message in the payload — omitting it triggers:
+    #     HTTP 400: "The reasoning_content in the thinking mode must be passed
+    #                back to the API"
+    #
+    # The canonical history contains assistant messages from two sources:
+    #   1. Real LLM responses — carry reasoning_content when returned by the
+    #      provider (preserved in agent.rb via parse_response).
+    #   2. Synthetic / locally-injected messages — skill injection, subagent
+    #      acks, slash-command notices, truncation fallbacks. These are never
+    #      produced by the LLM so they naturally lack reasoning_content.
+    #
+    # RULE: If ANY assistant message in the history carries reasoning_content,
+    # the conversation is provably running against a thinking-mode provider
+    # (the provider itself produced it). In that case, every other assistant
+    # message must echo the field, so we pad with an empty string.
+    #
+    # This is a purely structural inference with no model-name coupling —
+    # it self-adapts to new thinking-mode providers and new synthetic-message
+    # injection sites without any code changes elsewhere.
+    #
+    # For non-thinking providers (Claude / OpenAI / Gemini / Bedrock) no
+    # assistant message ever has reasoning_content, so this is a no-op.
+    # The Anthropic adapter also filters unknown fields via a whitelist, so
+    # even mid-session fallback between providers remains safe.
+    private def ensure_reasoning_content_consistency(msgs, force: false)
+      self.class.pad_reasoning_content_if_needed(msgs, force: force)
+    end
+
+    # Public helper: pad assistant messages that lack a reasoning_content
+    # field with an empty string, either when forced or when the payload
+    # already shows evidence of thinking-mode (at least one assistant
+    # message with reasoning_content).
+    #
+    # Exposed as a class method so Time Machine's active_messages path can
+    # reuse the exact same logic without routing through #to_api.
+    def self.pad_reasoning_content_if_needed(msgs, force: false)
+      should_pad = force || msgs.any? { |m| m[:role] == "assistant" && m[:reasoning_content] }
+      return msgs unless should_pad
+
+      msgs.map do |m|
+        next m unless m[:role] == "assistant"
+        next m if m.key?(:reasoning_content)
+
+        m.merge(reasoning_content: "")
+      end
+    end
+
+    # Defense-in-depth: recursively scrub invalid UTF-8 bytes from every String
+    # stored in the message tree. Even if a tool forgets to scrub its output,
+    # nothing poisoned will ever reach session persistence or JSON.generate.
+    #
+    # Fast path: if the tree contains only valid UTF-8 strings, the original
+    # object is returned unchanged — preserving object identity for callers
+    # that rely on `equal?` (e.g. rollback_before).
+    # Slow path: any invalid byte triggers a rebuild with scrubbed strings
+    # (invalid bytes → U+FFFD).
+    private def deep_sanitize_utf8(obj)
+      case obj
+      when String
+        return obj if obj.encoding == Encoding::UTF_8 && obj.valid_encoding?
+        obj.encode("UTF-8", invalid: :replace, undef: :replace, replace: "\u{FFFD}")
+      when Hash
+        return obj unless contains_dirty_utf8?(obj)
+        obj.transform_values { |v| deep_sanitize_utf8(v) }
+      when Array
+        return obj unless contains_dirty_utf8?(obj)
+        obj.map { |v| deep_sanitize_utf8(v) }
+      else
+        obj
+      end
+    end
+
+    # Cheap recursive check: does this subtree contain any invalid-UTF-8 string?
+    # Short-circuits on first offender. Keeps the common case (all valid UTF-8)
+    # allocation-free.
+    private def contains_dirty_utf8?(obj)
+      case obj
+      when String
+        !(obj.encoding == Encoding::UTF_8 && obj.valid_encoding?)
+      when Hash
+        obj.any? { |_, v| contains_dirty_utf8?(v) }
+      when Array
+        obj.any? { |v| contains_dirty_utf8?(v) }
+      else
+        false
+      end
+    end
+  end
+end

data/lib/octo/openai_stream_aggregator.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Octo
+  # Reassembles an OpenAI-compatible chat-completion event stream into the
+  # non-streaming response shape that MessageFormat::OpenAI.parse_response
+  # consumes, while invoking on_chunk(input_tokens:, output_tokens:) every
+  # time the upstream emits a new usage frame.
+  #
+  # Streaming frames look like:
+  #
+  #   {"id":"...","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}
+  #   {"id":"...","choices":[{"index":0,"delta":{"content":"Hi"},"finish_reason":null}]}
+  #   {"id":"...","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_x","function":{"name":"shell","arguments":"{\"cmd"}}]}}]}
+  #   {"id":"...","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\"ls\"}"}}]}}]}
+  #   {"id":"...","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}
+  #   {"id":"...","choices":[],"usage":{"prompt_tokens":12,"completion_tokens":3,"prompt_tokens_details":{"cached_tokens":2}}}
+  #   data: [DONE]
+  class OpenAIStreamAggregator
+    def initialize(on_chunk: nil)
+      @on_chunk = on_chunk
+      @content = +""
+      @reasoning_content = +""
+      @role = "assistant"
+      @finish_reason = nil
+      @tool_calls = {}
+      @usage = nil
+      @last_input_tokens = 0
+      @last_output_tokens = 0
+    end
+
+    def handle(data_str)
+      return if data_str == "[DONE]"
+      data = parse_or_nil(data_str)
+      return unless data
+
+      if (choice = (data["choices"] || []).first)
+        delta = choice["delta"] || {}
+        @role = delta["role"] if delta["role"]
+        @content << delta["content"] if delta["content"].is_a?(String)
+        @reasoning_content << delta["reasoning_content"] if delta["reasoning_content"].is_a?(String)
+        if (tcs = delta["tool_calls"])
+          tcs.each { |tc| merge_tool_call(tc) }
+        end
+        @finish_reason = choice["finish_reason"] if choice["finish_reason"]
+        emit_estimate_progress
+      end
+
+      if (u = data["usage"])
+        @usage = u
+        emit_usage_progress(u)
+      end
+    end
+
+    # Render the canonical non-streaming response shape.
+    def to_h
+      tool_calls = @tool_calls.keys.sort.map do |idx|
+        tc = @tool_calls[idx]
+        {
+          "id"       => tc[:id],
+          "type"     => tc[:type] || "function",
+          "function" => {
+            "name"      => tc[:name],
+            "arguments" => tc[:arguments].to_s
+          }
+        }
+      end
+
+      message = {
+        "role"    => @role,
+        "content" => @content.empty? ? nil : @content
+      }
+      message["tool_calls"] = tool_calls unless tool_calls.empty?
+      message["reasoning_content"] = @reasoning_content unless @reasoning_content.empty?
+
+      {
+        "choices" => [{ "index" => 0, "message" => message, "finish_reason" => @finish_reason }],
+        "usage"   => @usage || {}
+      }
+    end
+
+    private def merge_tool_call(tc)
+      idx = tc["index"] || @tool_calls.size
+      slot = (@tool_calls[idx] ||= { id: nil, type: nil, name: nil, arguments: +"" })
+      slot[:id] ||= tc["id"] if tc["id"]
+      slot[:type] ||= tc["type"] if tc["type"]
+      if (fn = tc["function"])
+        slot[:name] ||= fn["name"] if fn["name"]
+        slot[:arguments] << fn["arguments"].to_s if fn["arguments"]
+      end
+    end
+
+    private def parse_or_nil(s)
+      JSON.parse(s)
+    rescue JSON::ParserError
+      nil
+    end
+
+    private def emit_estimate_progress
+      return unless @on_chunk
+      output = approximate_output_tokens
+      return if output == @last_output_tokens
+      @last_output_tokens = output
+      @on_chunk.call(input_tokens: @last_input_tokens, output_tokens: output)
+    rescue => e
+      Octo::Logger.warn("[OpenAIStreamAggregator] on_chunk: #{e.class}: #{e.message}")
+    end
+
+    # Rough char/4 estimate; replaced by the real count when the upstream
+    # finally emits a usage frame (with stream_options.include_usage=true).
+    private def approximate_output_tokens
+      total_chars = @content.bytesize + @reasoning_content.bytesize +
+        @tool_calls.values.sum { |tc| tc[:arguments].to_s.bytesize }
+      (total_chars / 4.0).ceil
+    end
+
+    private def emit_usage_progress(u)
+      return unless @on_chunk
+      total_prompt = u["prompt_tokens"].to_i
+      output       = u["completion_tokens"].to_i
+      return if total_prompt == @last_input_tokens && output == @last_output_tokens
+      @last_input_tokens = total_prompt
+      @last_output_tokens = output
+      @on_chunk.call(input_tokens: total_prompt, output_tokens: output)
+    rescue => e
+      Octo::Logger.warn("[OpenAIStreamAggregator] on_chunk: #{e.class}: #{e.message}")
+    end
+  end
+end

data/lib/octo/plain_ui_controller.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require_relative "ui_interface"
+
+module Octo
+  # PlainUIController implements UIInterface for non-interactive (--message) mode.
+  # Writes human-readable plain text directly to stdout so the caller can capture
+  # or pipe the output. No spinners, no TUI — just clean lines.
+  class PlainUIController
+    include Octo::UIInterface
+
+    def initialize(output: $stdout)
+      @output = output
+      @mutex = Mutex.new
+    end
+
+    # === Output display ===
+
+    def show_assistant_message(content, files:)
+      puts_line(content) unless content.nil? || content.strip.empty?
+      files.each { |f| puts_line("📄 File: #{f[:path]}") }
+    end
+
+    def show_tool_call(name, args)
+      args_data = args.is_a?(String) ? (JSON.parse(args) rescue args) : args
+
+      # Special handling for request_user_feedback — display as a readable prompt
+      if name.to_s == "request_user_feedback"
+        question = args_data.is_a?(Hash) ? (args_data[:question] || args_data["question"]).to_s : ""
+        context  = args_data.is_a?(Hash) ? (args_data[:context]  || args_data["context"]).to_s  : ""
+        options  = args_data.is_a?(Hash) ? (args_data[:options]  || args_data["options"])        : nil
+        options  = Array(options) if options && !options.is_a?(Array)
+
+        parts = []
+        parts << "**Context:** #{context.strip}" if context && !context.strip.empty?
+        parts << "**Question:** #{question.strip}"
+        if options && !options.empty?
+          parts << "**Options:**"
+          options.each_with_index { |opt, i| parts << "  #{i + 1}. #{opt}" }
+        end
+        puts_line(parts.join("\n"))
+        return
+      end
+
+      display = case name
+                when "terminal"
+                  cmd = args_data.is_a?(Hash) ? (args_data[:command] || args_data["command"]) : args_data
+                  sid = args_data.is_a?(Hash) ? (args_data[:session_id] || args_data["session_id"]) : nil
+                  if cmd
+                    "$ #{cmd}"
+                  elsif sid
+                    "$ (session ##{sid})"
+                  else
+                    "$ terminal"
+                  end
+                when "write"
+                  path = args_data.is_a?(Hash) ? (args_data[:path] || args_data["path"]) : args_data
+                  "Write → #{path}"
+                when "edit"
+                  path = args_data.is_a?(Hash) ? (args_data[:path] || args_data["path"]) : args_data
+                  "Edit → #{path}"
+                else
+                  label = args_data.is_a?(Hash) ? args_data.map { |k, v| "#{k}=#{v.to_s[0..40]}" }.join(", ") : args_data.to_s[0..80]
+                  "#{name}(#{label})"
+                end
+      puts_line("[tool] #{display}")
+    end
+
+    def show_tool_result(result)
+      text = result.to_s.strip
+      return if text.empty?
+
+      # Indent multi-line results for readability
+      indented = text.lines.map { |l| "  #{l}" }.join
+      puts_line(indented)
+    end
+
+    def show_tool_error(error)
+      msg = error.is_a?(Exception) ? error.message : error.to_s
+      puts_line("[error] #{msg}")
+    end
+
+    def show_file_write_preview(path, is_new_file:)
+      action = is_new_file ? "create" : "overwrite"
+      puts_line("[file] #{action}: #{path}")
+    end
+
+    def show_file_edit_preview(path)
+      puts_line("[file] edit: #{path}")
+    end
+
+    def show_file_error(error_message)
+      puts_line("[file error] #{error_message}")
+    end
+
+    def show_shell_preview(command)
+      puts_line("[shell] #{command}")
+    end
+
+    def show_complete(iterations:, duration: nil, cache_stats: nil, awaiting_user_feedback: false)
+      parts = ["[done] iterations=#{iterations}"]
+      parts << "duration=#{duration.round(1)}s" if duration
+      puts_line(parts.join(" "))
+    end
+
+    def append_output(content)
+      puts_line(content)
+    end
+
+    # === Status messages ===
+
+    def show_info(message, prefix_newline: true)
+      puts_line("[info] #{message}")
+    end
+
+    def show_warning(message)
+      puts_line("[warn] #{message}")
+    end
+
+    def show_error(message)
+      puts_line("[error] #{message}")
+    end
+
+    def show_success(message)
+      puts_line("[ok] #{message}")
+    end
+
+    def log(message, level: :info)
+      # Only surface errors/warnings; suppress debug noise in plain mode
+      puts_line("[#{level}] #{message}") if %i[error warn].include?(level.to_sym)
+    end
+
+    # === Progress (no-ops — no spinner in plain mode) ===
+
+    def show_progress(message = nil, prefix_newline: true, progress_type: "thinking", phase: "active", metadata: {}); end
+
+    # === State updates (no-ops) ===
+
+    def update_sessionbar(tasks: nil, status: nil, latency: nil); end
+    def update_todos(todos); end
+    def set_working_status; end
+    def set_idle_status; end
+
+    # === Blocking interaction (auto-approve in non-interactive mode) ===
+
+    def request_confirmation(message, default: true)
+      # Should not be reached because permission_mode is forced to auto_approve,
+      # but return true as a safety net.
+      true
+    end
+
+    # === Input control / Lifecycle (no-ops) ===
+
+    def clear_input; end
+    def set_input_tips(message, type: :info); end
+    def stop; end
+
+
+    def puts_line(text)
+      @mutex.synchronize do
+        @output.puts(text)
+        @output.flush
+      end
+    end
+  end
+end