llm.rb 12.2.0 → 12.3.1

data/lib/llm/providers/bedrock.rb

@@ -205,9 +205,8 @@ module LLM
       params = {role: :user, model: default_model, max_tokens: 2048}.merge!(params)
       tools = resolve_tools(params.delete(:tools))
       params = [params, adapt_schema(params), adapt_tools(tools)].inject({}, &:merge!).compact
-      role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if streamable?(stream) || stream == true
-      [params, stream, tools, role]
+      role, stream = params.delete(:role), LLM::Stream.try(params.delete(:stream))
+      [params.merge!(stream: stream.enabled?), stream, tools, role]
     end
 
     def build_complete_request(prompt, params, role, stream: nil)
@@ -215,8 +214,8 @@ module LLM
       model_id = params.delete(:model) || default_model
       payload = build_converse_payload(messages, params)
       body = LLM.json.dump(payload)
-      path = stream ? "/model/#{model_id}/converse-stream" \
-                    : "/model/#{model_id}/converse"
+      path = stream&.enabled? ? "/model/#{model_id}/converse-stream" \
+                              : "/model/#{model_id}/converse"
       req = LLM::Transport::Request.post(path, headers)
       transport.set_body_stream(req, StringIO.new(body))
       [req, messages, body]

data/lib/llm/providers/google.rb

@@ -199,8 +199,10 @@ module LLM
       tools  = resolve_tools(params.delete(:tools))
       config = adapt_generation_config(params.except(*except))
       params = [params.except(:schema), config, adapt_tools(tools)].inject({}, &:merge!).compact
-      role, model, stream = [:role, :model, :stream].map { params.delete(_1) }
-      [params, stream, tools, role, model]
+      role, model, stream = params.delete(:role),
+                            params.delete(:model),
+                            LLM::Stream.try(params.delete(:stream))
+      [params.merge!(stream: stream.enabled?), stream, tools, role, model]
     end
 
     def build_complete_request(prompt, params, role, model, stream)

data/lib/llm/providers/mistral.rb

@@ -56,6 +56,32 @@ module LLM
       super
     end
 
+    ##
+    # Runs OCR on a remote image or document URL.
+    # @see https://docs.mistral.ai/api/endpoint/ocr#operation-ocr_v1_ocr_post Mistral OCR docs
+    # @param [String, nil] image_url
+    #  A remote HTTP(S) URL to the image
+    # @param [String, nil] document_url
+    #  A remote HTTP(S) URL to the document
+    # @param [String] model
+    #  The OCR model to use
+    # @param [Hash] params
+    #  Additional OCR parameters
+    # @raise (see LLM::Provider#request)
+    # @return [LLM::Response]
+    def ocr(image_url: nil, document_url: nil, model: "mistral-ocr-latest", **params)
+      if [image_url, document_url].all?(&:nil?)
+        raise ArgumentError, "must provide one of: image_url, document_url"
+      elsif [image_url, document_url].compact.size > 1
+        raise ArgumentError, "must provide one of: image_url, document_url"
+      end
+      document = parse_document(image_url, document_url)
+      req = LLM::Transport::Request.post("/v1/ocr", headers)
+      req.body = LLM.json.dump({model:, document:}.merge!(params))
+      res, = execute(request: req, operation: "ocr", model:)
+      LLM::Response.new(res)
+    end
+
     ##
     # @raise [NotImplementedError]
     def responses
@@ -92,5 +118,28 @@ module LLM
     def default_model
       "mistral-large-latest"
     end
+
+    private
+
+    ##
+    # @api private
+    def headers
+      lock do
+        (@headers || {}).merge(
+          "Authorization" => "Bearer #{@key}",
+          "Content-Type" => "application/json"
+        )
+      end
+    end
+
+    ##
+    # @api private
+    def parse_document(image_url, document_url)
+      if image_url
+        {type: "image_url", image_url:}
+      elsif document_url
+        {type: "document_url", document_url:}
+      end
+    end
   end
 end

data/lib/llm/providers/ollama/response_adapter/completion.rb

@@ -9,6 +9,16 @@ module LLM::Ollama::ResponseAdapter
     end
     alias_method :choices, :messages
 
+    ##
+    # Returns the response body, parsing NDJSON when the
+    # transport returned a raw string (non-streaming path).
+    # @return [LLM::Object]
+    def body
+      raw = super
+      return raw unless String === raw
+      parse_ndjson(raw)
+    end
+
     ##
     # (see LLM::Contract::Completion#input_tokens)
     def input_tokens
@@ -110,6 +120,35 @@ module LLM::Ollama::ResponseAdapter
       end
     end
 
+    ##
+    # Parses a raw NDJSON string into an LLM::Object.
+    # Ollama's streaming API returns application/x-ndjson,
+    # but the transport only parses application/json. When
+    # the response body arrives as a raw string, we merge
+    # the NDJSON lines here.
+    # @param [String] raw
+    # @return [LLM::Object]
+    def parse_ndjson(raw)
+      lines = raw.split("\n").reject(&:empty?)
+      merged = lines.each_with_object({}) do |line, hash|
+        parsed = LLM.json.load(line)
+        next unless parsed.is_a?(Hash)
+        parsed.each do |key, value|
+          if key == "message" && value.is_a?(Hash)
+            if hash.key?("message")
+              hash["message"]["content"] << value["content"].to_s
+            else
+              hash["message"] = {"role" => value["role"], "content" => value["content"].to_s}
+              hash["message"]["tool_calls"] = value["tool_calls"] if value.key?("tool_calls")
+            end
+          else
+            hash[key] = value
+          end
+        end
+      end
+      LLM::Object.from(merged)
+    end
+
     include LLM::Contract::Completion
   end
 end

data/lib/llm/providers/ollama.rb

@@ -121,9 +121,8 @@ module LLM
       params = {role: :user, model: default_model, stream: true}.merge!(params)
       tools  = resolve_tools(params.delete(:tools))
       params = [params, {format: params[:schema]}, adapt_tools(tools)].inject({}, &:merge!).compact
-      role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if streamable?(stream) || stream == true
-      [params, stream, tools, role]
+      role, stream = params.delete(:role), LLM::Stream.try(params.delete(:stream))
+      [params.merge!(stream: stream.enabled?), stream, tools, role]
     end
 
     def build_complete_request(prompt, params, role)

data/lib/llm/providers/openai/responses.rb

@@ -36,10 +36,13 @@ class LLM::OpenAI
     # @return [LLM::Response]
     def create(prompt, params = {})
       params = {role: :user, model: @provider.default_model}.merge!(params)
+      role, stream = params.delete(:role), LLM::Stream.try(params.delete(:stream))
       tools  = resolve_tools(params.delete(:tools))
-      params = [params, adapt_schema(params), adapt_tools(tools)].inject({}, &:merge!).compact
-      role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if @provider.streamable?(stream) || stream == true
+      params = [
+        params.merge!(stream: stream.enabled?),
+        adapt_schema(params),
+        adapt_tools(tools)
+      ].inject({}, &:merge!).compact
       req = LLM::Transport::Request.post(path("/responses"), headers)
       messages = build_complete_messages(prompt, params, role)
       @provider.tracer.set_request_metadata(user_input: extract_user_input(messages, fallback: prompt))

data/lib/llm/providers/openai.rb

@@ -211,12 +211,11 @@ module LLM
       params = {role: :user, model: default_model}.merge!(params)
       tools = resolve_tools(params.delete(:tools))
       params = [params, adapt_schema(params), adapt_tools(tools)].inject({}, &:merge!).compact
-      role, stream = params.delete(:role), params.delete(:stream)
-      params[:stream] = true if streamable?(stream) || stream == true
+      role, stream = params.delete(:role), LLM::Stream.try(params.delete(:stream))
       if params[:stream]
         params[:stream_options] = {include_usage: true}.merge!(params[:stream_options] || {})
       end
-      [params, stream, tools, role]
+      [params.merge!(stream: stream.enabled?), stream, tools, role]
     end
 
     def build_complete_request(prompt, params, role)

data/lib/llm/repl/bar.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+class LLM::Repl
+  ##
+  # The {LLM::Repl::Bar LLM::Repl::Bar} class renders a
+  # small progress bar for the REPL. It is used to show
+  # the remaining size of the model's context window in
+  # a compact form near the input line.
+  # @api private
+  class Bar
+    ##
+    # @return [String]
+    OCCUPIED = "█"
+
+    ##
+    # @return [String]
+    FREE = " "
+
+    ##
+    # @param [Integer] used
+    # @param [Integer] total
+    # @param [Integer] width
+    # @return [LLM::Repl::Bar]
+    def initialize(used:, total:, width: 10)
+      @width = width
+      @label, @filled = remainder(used, total)
+    end
+
+    ##
+    # @return [String]
+    def to_s
+      bar = "#{OCCUPIED * filled}#{FREE * (width - filled)}"
+      "│#{bar}│ #{label}"
+    end
+
+    private
+
+    ##
+    # @param [Integer] used
+    # @param [Integer] total
+    # @return [[String, Integer]]
+    def remainder(used, total)
+      return ["???", width] if total <= 0
+      diff = total - used
+      return ["0%", 0] if diff <= 0
+      remaining = (diff.fdiv(total) * 100).round(2)
+      ["#{remaining}%", ((remaining / 100) * width).round]
+    end
+
+    attr_reader :label, :filled, :width
+  end
+end

data/lib/llm/repl/input.rb

@@ -6,59 +6,172 @@ class LLM::Repl
   # the editable input line shown at the bottom of the REPL.
   # @api private
   class Input
+    CTRL_A    = 1
+    CTRL_E    = 5
     UP        = Curses::Key::UP
     DOWN      = Curses::Key::DOWN
+    LEFT      = Curses::Key::LEFT
+    RIGHT     = Curses::Key::RIGHT
     ENTER     = [Curses::Key::ENTER, 10, 13]
     BACKSPACE = [Curses::Key::BACKSPACE, 127]
-    EOF       = [nil, 4]
+    EOF       = [4]
 
     ##
-    # @param [String, Symbol] provider
+    # @param [LLM::Agent] agent
     # @return [LLM::Repl::Input]
-    def initialize(provider)
-      @provider = provider
+    def initialize(agent, options = {})
+      @agent = agent
+      @provider = agent.llm.name
       @buffer = +""
+      @cursor = 0
+      @scroll = 0
+      @height = options.fetch(:height, 3)
     end
 
     ##
     # @param [LLM::Repl::Window] window
-    # @return [String, nil]
-    def readline(window)
-      catch(:done) do
-        @buffer.clear
-        loop do
-          on_char(window, window.getch)
-          window.redraw
-        end
-      end
-    end
-
-    ##
-    # @return [String]
-    def to_s
-      "> #{@buffer}"
-    end
-
-    private
-
+    # @param [Object] char
+    # @return [Symbol, nil]
     def on_char(window, char)
       if EOF.include?(char)
-        throw(:done, nil)
+        :exit
       elsif BACKSPACE.include?(char)
-        @buffer.chop!
+        backspace
+        :backspace
       elsif ENTER.include?(char)
-        buf = @buffer.dup
-        @buffer.clear
-        throw(:done, buf)
+        :submit
       elsif char == UP
         window.scroll_up
+        :up
       elsif char == DOWN
         window.scroll_down
+        :down
+      elsif char == CTRL_A
+        move_start
+        :ctrl_a
+      elsif char == CTRL_E
+        move_end
+        :ctrl_e
+      elsif char == LEFT
+        move_left
+        :left
+      elsif char == RIGHT
+        move_right
+        :right
       elsif String === char
-        @buffer << char
+        insert(char)
+        :char
       else
-        # ???
+        nil
       end
     end
+
+    ##
+    # @return [String]
+    def to_s
+      "#{@provider}> #{@buffer}"
+    end
+
+    ##
+    # @return [Integer]
+    def cursor
+      prompt.length + @cursor
+    end
+
+    ##
+    # @return [Integer]
+    def height
+      @height
+    end
+
+    ##
+    # Returns the visible lines of the input buffer,
+    # wrapped at the given column width. The viewport
+    # follows the cursor so the cursor line is always
+    # visible.
+    # @param [Integer] cols
+    # @return [Array<String>]
+    def lines(cols)
+      sync_scroll(cols)
+      text = to_s
+      chunks = text.chars.each_slice(cols).map(&:join)
+      chunks = [""] if chunks.empty?
+      chunks[@scroll, height] || []
+    end
+
+    ##
+    # Returns the cursor position as [line, column] within
+    # the visible viewport.
+    # @param [Integer] cols
+    # @return [Array(Integer, Integer)]
+    def cursor_pos(cols)
+      sync_scroll(cols)
+      [(cursor / cols) - @scroll, cursor % cols]
+    end
+
+    ##
+    # @return [void]
+    def move_start
+      @cursor = 0
+    end
+
+    ##
+    # @return [void]
+    def move_end
+      @cursor = [0, @buffer.size].max
+    end
+
+    ##
+    # @return [void]
+    def move_left
+      @cursor = [@cursor - 1, 0].max
+    end
+
+    ##
+    # @return [void]
+    def move_right
+      @cursor = [@cursor + 1, @buffer.size].min
+    end
+
+    ##
+    # @return [String]
+    def take
+      @buffer.dup.tap do
+        @buffer.clear
+        @cursor = 0
+        @scroll = 0
+      end
+    end
+
+    private
+
+    ##
+    # Adjusts @scroll so the cursor line is visible within
+    # the viewport.
+    def sync_scroll(cols)
+      total_lines = [1, (to_s.length.to_f / cols).ceil].max
+      cursor_line = cursor / cols
+      if cursor_line < @scroll
+        @scroll = cursor_line
+      elsif cursor_line >= (@scroll + height)
+        @scroll = (cursor_line - height) + 1
+      end
+      @scroll = [[@scroll, (total_lines - height)].min, 0].max
+    end
+
+    def prompt
+      "#{@provider}> "
+    end
+
+    def insert(char)
+      @buffer.insert(@cursor, char)
+      @cursor += char.length
+    end
+
+    def backspace
+      return if @cursor <= 0
+      @buffer.slice!(@cursor - 1)
+      @cursor -= 1
+    end
   end
 end

data/lib/llm/repl/markdown.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+class LLM::Repl
+  ##
+  # This class is designed to represent a markdown
+  # string (typically from a model's response) as a
+  # tree of objects where each object contains a piece
+  # of text, and also optional style information for
+  # that text (eg bold, underscore, ...)
+  class Markdown
+    ##
+    # @param [String] text
+    # @return [LLM::Repl::Markdown]
+    def initialize(text)
+      @doc = Kramdown::Document.new(text)
+      @ast = []
+    end
+
+    ##
+    # @return [Array<Hash>]
+    def ast
+      @ast.tap do
+        ##
+        # Recurisvely travels the markdown document and
+        # populates the `@ast` variable along the way.
+        # The AST is composed of structured data that
+        # carries both text and styling information that
+        # is applied by the UI thread.
+        walk(@doc.root)
+
+        ##
+        # This is required because the AST collects
+        # empty nodes towards the end of the tree.
+        # If we don't pop them we end up with excessive
+        # amount of newlines between turns.
+        last = @ast.last
+        while last and last[:text].to_s.strip.empty?
+          @ast.pop
+          last = @ast.last
+        end
+      end
+    end
+
+    private
+
+    ##
+    # Recursively walk from the head node to the
+    # tail node. This method mutates the `@ast`
+    # variable. A future refactor might be worthwhile
+    # since this method is implemented with side effects,
+    # but it probably could return the ast instead.
+    def walk(node, attrs = nil)
+      case node.type
+      when :root
+        node.children.each { walk(_1, attrs) }
+      when :text
+        emit(node.value.to_s, attrs)
+      when :p
+        node.children.each { walk(_1, attrs) }
+        emit("\n\n", attrs)
+      when :header
+        emit("\n", attrs)
+        node.children.each { walk(_1, Curses::A_BOLD) }
+        emit("\n", attrs)
+      when :strong
+        node.children.each { walk(_1, Curses::A_BOLD) }
+      when :em
+        node.children.each { walk(_1, Curses::A_UNDERLINE) }
+      when :codespan
+        emit(node.value, Curses::A_REVERSE)
+      when :codeblock
+        emit(node.value, Curses::A_REVERSE)
+        emit("\n\n", attrs)
+      when :br
+        emit("\n", attrs)
+      else
+        node.children.each { walk(_1, attrs) }
+      end
+    end
+
+    def emit(text, attrs)
+      @ast.push({text: text.to_s, attrs:}.compact)
+    end
+  end
+end

data/lib/llm/repl/status.rb

@@ -9,18 +9,29 @@ class LLM::Repl
     ##
     # @param [String, Symbol] provider
     # @return [LLM::Repl::Status]
-    def initialize(provider)
-      @provider = provider
+    def initialize(agent)
+      @agent = agent
+      @provider = agent.llm.name
       @text = "idle"
     end
 
     ##
     # @return [String]
-    attr_reader :provider
+    def context_bar
+      LLM::Repl::Bar.new(
+        used: @agent.usage.total_tokens,
+        total: @agent.context_window
+      ).to_s
+    end
+
+    ##
+    # @return [String]
+    def cost
+      "$#{@agent.cost}"
+    end
 
     ##
-    # @param [String] value
-    # @return [void]
+    # @return [String]
     attr_accessor :text
 
     ##

data/lib/llm/repl/stream.rb

@@ -11,8 +11,10 @@ class LLM::Repl
     ##
     # @param [LLM::Repl] repl
     # @return [LLM::Repl::Stream]
-    def initialize(repl)
+    def initialize(repl, queue)
       @repl = repl
+      @_queue = queue
+      @buffer = +""
     end
 
     ##
@@ -20,7 +22,8 @@ class LLM::Repl
     #  One or more chars
     # @return [void]
     def on_content(chars)
-      @repl.write(chars)
+      @buffer << chars
+      @_queue.push [:stream, @buffer]
     end
 
     ##
@@ -29,9 +32,9 @@ class LLM::Repl
     # @return [void]
     def on_tool_call(tool, error)
       if error
-        @repl.status = "tool error: #{tool.name}"
+        @_queue.push [:status, "tool not found: #{tool.name}"]
       else
-        @repl.status = "tool: #{tool.name}"
+        @_queue.push [:status, "tool: #{tool.name}"]
       end
     end
 
@@ -40,7 +43,14 @@ class LLM::Repl
     # @param [LLM::Function::Return] result
     # @return [void]
     def on_tool_return(_tool, result)
-      @repl.status = "tool done: #{result.name}"
+      @_queue.push [:status, "tool done: #{result.name}"]
+    end
+
+    ##
+    # Empty the accumulated buffer
+    # @return [void]
+    def empty!
+      @buffer.clear
     end
   end
 end