llm.rb 7.0.0 → 8.1.0

data/lib/llm/providers/bedrock/stream_decoder.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "digest"
+
+class LLM::Bedrock
+  ##
+  # Decodes AWS Event Stream binary frames.
+  #
+  # Bedrock Converse Stream uses the AWS Event Stream protocol,
+  # a binary framing format (not SSE). Each message has:
+  #   - total length (4 bytes, big-endian)
+  #   - headers length (4 bytes, big-endian)
+  #   - prelude CRC (4 bytes)
+  #   - headers (variable)
+  #   - payload (variable, usually JSON)
+  #   - message CRC (4 bytes)
+  #
+  # Implements #<< to match the interface expected by llm.rb's
+  # streaming transport, so it can replace the SSE-based
+  # StreamDecoder when streaming from Bedrock.
+  #
+  # @api private
+  class StreamDecoder
+    ##
+    # @return [LLM::Bedrock::StreamParser]
+    attr_reader :parser
+
+    ##
+    # @param [LLM::Bedrock::StreamParser] parser
+    def initialize(parser)
+      @buffer = +"".b
+      @parser = parser
+    end
+
+    ##
+    # Feeds a raw binary chunk into the decoder.
+    # Accumulates data until complete frames are available,
+    # then decodes them and passes the JSON payload to the parser.
+    #
+    # @param [String] chunk Raw binary data
+    # @return [void]
+    def <<(chunk)
+      @buffer << chunk
+      decode_frames
+    end
+
+    ##
+    # @return [Hash] The fully constructed response body
+    def body
+      parser.body
+    end
+
+    ##
+    # @return [void]
+    def free
+      @buffer.clear
+      parser.free if parser.respond_to?(:free)
+    end
+
+    private
+
+    def decode_frames
+      loop do
+        break if @buffer.bytesize < 12
+        total_length = @buffer[0, 4].unpack1("N")
+        break if @buffer.bytesize < total_length
+        # headers_length = @buffer[4, 4].unpack1("N")
+        # prelude_crc = @buffer[8, 4].unpack1("N")
+        headers = decode_headers
+        payload_start = 12 + headers[:length]
+        payload_length = total_length - payload_start - 4
+        payload = @buffer[payload_start, payload_length] if payload_length > 0
+        # message_crc from last 4 bytes, not needed for our purposes
+        json = payload ? LLM.json.load(payload) : {}
+        parser.parse!(json, event_type: headers[:event_type]) if json.is_a?(Hash)
+        @buffer = @buffer[total_length..] || +"".b
+      end
+    end
+
+    def decode_headers
+      headers_length = @buffer[4, 4].unpack1("N")
+      offset = 12
+      end_offset = offset + headers_length
+      result = {event_type: nil, length: headers_length}
+      while offset < end_offset
+        name_len = @buffer.getbyte(offset)
+        offset += 1
+        break if offset + name_len > end_offset
+        name = @buffer[offset, name_len]
+        offset += name_len
+        break if offset >= end_offset
+        value_type = @buffer.getbyte(offset)
+        offset += 1
+        value = case value_type
+        when 7 # string
+          str_len = @buffer[offset, 2].unpack1("n")
+          offset += 2
+          str = @buffer[offset, str_len]
+          offset += str_len
+          str
+        when 8 # binary
+          bin_len = @buffer[offset, 2].unpack1("n")
+          offset += 2
+          bin = @buffer[offset, bin_len]
+          offset += bin_len
+          bin
+        when 9 # boolean true
+          true
+        when 1 # boolean false
+          false
+        when 2 # byte
+          val = @buffer.getbyte(offset)
+          offset += 1
+          val
+        when 3 # int16
+          val = @buffer[offset, 2].unpack1("s>")
+          offset += 2
+          val
+        when 4 # int32
+          val = @buffer[offset, 4].unpack1("l>")
+          offset += 4
+          val
+        when 6 # byte array (as raw string)
+          bin_len = @buffer[offset, 2].unpack1("n")
+          offset += 2
+          bin = @buffer[offset, bin_len]
+          offset += bin_len
+          bin
+        else
+          # Unknown type, skip to end of headers
+          offset = end_offset
+          nil
+        end
+        result[:event_type] = value if name == ":event-type"
+        result[name] = value if name
+      end
+      result
+    end
+  end
+end

data/lib/llm/providers/bedrock/stream_parser.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+class LLM::Bedrock
+  ##
+  # Parses Bedrock Converse Stream events into a response body
+  # and emits stream callbacks (on_content, on_tool_call, etc.).
+  #
+  # Receives decoded JSON payloads from {StreamDecoder} along with
+  # the AWS Event Stream event type header.
+  #
+  # Bedrock Converse Stream event types:
+  #   messageStart — initial role
+  #   contentBlockStart — tool use or reasoning start
+  #   contentBlockDelta — text delta, tool input JSON, or reasoning text
+  #   contentBlockStop — content block finished
+  #   messageStop — final stop reason, usage metadata
+  #
+  # @api private
+  class StreamParser
+    TOOL_MARKER = "<｜DSML｜function_calls"
+
+    ##
+    # @return [Hash] Fully constructed response body
+    attr_reader :body
+
+    ##
+    # @param [#<<, LLM::Stream] stream
+    def initialize(stream)
+      @body = {"output" => {"message" => {"role" => "assistant", "content" => []}}}
+      @stream = stream
+      @text_markers = {}
+      @can_emit_content = stream.respond_to?(:on_content)
+      @can_emit_reasoning_content = stream.respond_to?(:on_reasoning_content)
+      @can_emit_tool_call = stream.respond_to?(:on_tool_call)
+      @can_push_content = stream.respond_to?(:<<)
+    end
+
+    ##
+    # @param [Hash] payload Decoded JSON from an event stream frame
+    # @param [String, nil] event_type The :event-type header value
+    # @return [self]
+    def parse!(payload, event_type: nil)
+      type = event_type || payload["type"]
+      case type
+      when "messageStart"
+        # { "role" => "assistant" }
+      when "contentBlockStart"
+        # { "contentBlockIndex" => 0, "start" => { "toolUse" => {...} } }
+        handle_content_block_start(payload)
+      when "contentBlockDelta"
+        # { "contentBlockIndex" => 0, "delta" => { "text" => "..." } }
+        handle_content_block_delta(payload)
+      when "contentBlockStop"
+        handle_content_block_stop(payload)
+      when "messageStop"
+        # { "stopReason" => "end_turn", "metadata" => {"usage" => {...}} }
+        merge_metadata(payload)
+      when "metadata"
+        # { "usage" => {...} }
+        merge_metadata(payload)
+      end
+      self
+    end
+
+    ##
+    # @return [void]
+    def free
+      @text_markers.clear
+    end
+
+    private
+
+    def handle_content_block_start(payload)
+      index = payload["contentBlockIndex"]
+      start_data = payload["start"] || {}
+      if (tool_use = start_data["toolUse"])
+        content[index] = {"toolUse" => {"toolUseId" => tool_use["toolUseId"], "name" => tool_use["name"], "input" => +""}}
+      elsif (reasoning = start_data["reasoningContent"])
+        content[index] = {"reasoningContent" => {"text" => +"", "signature" => reasoning["signature"]}.compact}
+      end
+    end
+
+    def handle_content_block_delta(payload)
+      index = payload["contentBlockIndex"]
+      delta = payload["delta"] || {}
+      if (text = delta["text"])
+        ensure_content_block(index, "text")
+        visible = filtered_text(index, text)
+        return if visible.empty?
+        content[index]["text"] ||= +""
+        content[index]["text"] << visible
+        emit_content(visible)
+      elsif (tool_input = delta.dig("toolUse", "input"))
+        ensure_content_block(index, "tool_use")
+        content[index]["toolUse"]["input"] ||= +""
+        content[index]["toolUse"]["input"] << tool_input
+      elsif (reasoning = delta["reasoningContent"])
+        ensure_content_block(index, "reasoning")
+        if reasoning["text"]
+          content[index]["reasoningContent"]["text"] ||= +""
+          content[index]["reasoningContent"]["text"] << reasoning["text"]
+          emit_reasoning_content(reasoning["text"])
+        end
+        if reasoning["signature"]
+          content[index]["reasoningContent"]["signature"] = reasoning["signature"]
+        end
+      end
+    end
+
+    def handle_content_block_stop(payload)
+      index = payload["contentBlockIndex"]
+      item = content[index]
+      return unless item
+      flush_text(index, item)
+      if item["toolUse"] && item["toolUse"]["input"].is_a?(String)
+        parsed = LLM.json.load(item["toolUse"]["input"])
+        item["toolUse"]["input"] = parsed.is_a?(Hash) ? parsed : {}
+        emit_tool(item)
+      end
+    rescue *LLM.json.parser_error
+      item["toolUse"]["input"] = {} if item&.dig("toolUse")
+    end
+
+    def ensure_content_block(index, type)
+      content[index] ||= case type
+      when "tool_use" then {"toolUse" => {"input" => +""}}
+      when "reasoning" then {"reasoningContent" => {"text" => +""}}
+      else {}
+      end
+    end
+
+    def filtered_text(index, text)
+      state = (@text_markers[index] ||= +"")
+      value = state << text
+      value.gsub!(TOOL_MARKER, "")
+      keep = marker_prefix_length(value)
+      @text_markers[index] = keep.zero? ? +"" : value[-keep..]
+      keep.zero? ? value : value[0...-keep]
+    end
+
+    def flush_text(index, item)
+      value = @text_markers.delete(index).to_s
+      return unless item["text"]
+      if value.empty?
+        content[index] = {} if item["text"].empty?
+      else
+        item["text"] << value
+        emit_content(value)
+      end
+    end
+
+    def marker_prefix_length(value)
+      [value.length, TOOL_MARKER.length - 1].min.downto(1) do |length|
+        return length if TOOL_MARKER.start_with?(value[-length..])
+      end
+      0
+    end
+
+    def merge_metadata(payload)
+      metadata = payload["metadata"] || payload
+      return unless metadata.is_a?(Hash)
+      usage = metadata["usage"]
+      @body["usage"] = usage if usage
+      @body["stopReason"] = payload["stopReason"] if payload["stopReason"]
+    end
+
+    def emit_content(value)
+      if @can_emit_content
+        @stream.on_content(value)
+      elsif @can_push_content
+        @stream << value
+      end
+    end
+
+    def emit_reasoning_content(value)
+      @stream.on_reasoning_content(value) if @can_emit_reasoning_content
+    end
+
+    def emit_tool(tool)
+      return unless @can_emit_tool_call
+      function, error = resolve_tool(tool)
+      @stream.on_tool_call(function, error)
+    end
+
+    def resolve_tool(tool)
+      payload = tool["toolUse"] || {}
+      registered = @stream.find_tool(payload["name"])
+      fn = (registered || LLM::Function.new(payload["name"])).dup.tap do |f|
+        f.id = payload["toolUseId"]
+        f.arguments = payload["input"] || {}
+        f.tracer = @stream.extra[:tracer]
+        f.model = @stream.extra[:model]
+      end
+      [fn, registered ? nil : @stream.tool_not_found(fn)]
+    end
+
+    def content
+      @body["output"]["message"]["content"]
+    end
+  end
+end

data/lib/llm/providers/bedrock.rb

@@ -0,0 +1,272 @@
+# frozen_string_literal: true
+
+module LLM
+  ##
+  # The Bedrock class implements a provider for
+  # [Amazon Bedrock](https://aws.amazon.com/bedrock/).
+  #
+  # Bedrock provides access to foundation models from Anthropic, Meta,
+  # Mistral, AI21 Labs, Cohere, and more through the AWS infrastructure.
+  # This provider uses the Bedrock Converse API for chat completions,
+  # and the Converse Stream API for streaming.
+  #
+  # Unlike other llm.rb providers which use API key authentication,
+  # Bedrock uses AWS Signature V4 (SigV4) for request signing.
+  # You must provide AWS credentials (access key, secret key, and region)
+  # instead of a single API key.
+  #
+  # Streaming uses the AWS Event Stream binary protocol instead of
+  # standard SSE. The binary framing is decoded inline using only
+  # Ruby's stdlib.
+  #
+  # @example
+  #   require "llm"
+  #
+  #   llm = LLM.bedrock(
+  #     access_key_id: ENV["AWS_ACCESS_KEY_ID"],
+  #     secret_access_key: ENV["AWS_SECRET_ACCESS_KEY"],
+  #     region: "us-east-1"
+  #   )
+  #   ctx = LLM::Context.new(llm)
+  #   ctx.talk "Hello, how are you?"
+  #   ctx.messages.select(&:assistant?).each { puts _1.content }
+  class Bedrock < Provider
+    require_relative "bedrock/signature"
+    require_relative "bedrock/error_handler"
+    require_relative "bedrock/request_adapter"
+    require_relative "bedrock/request_adapter/completion"
+    require_relative "bedrock/response_adapter"
+    require_relative "bedrock/response_adapter/completion"
+    require_relative "bedrock/response_adapter/models"
+    require_relative "bedrock/stream_decoder"
+    require_relative "bedrock/stream_parser"
+    require_relative "bedrock/models"
+
+    include RequestAdapter
+
+    HOST_PATTERN = "bedrock-runtime.%s.amazonaws.com"
+
+    ##
+    # @param [String] access_key_id AWS access key ID
+    # @param [String] secret_access_key AWS secret access key
+    # @param [String] region AWS region (e.g. "us-east-1")
+    # @param [String, nil] session_token AWS session token for temporary credentials
+    # @param [String, nil] host Override the Bedrock API host
+    # @param [Integer] port Connection port
+    # @param [Boolean] ssl Whether to use SSL
+    # @param [Integer] timeout Request timeout in seconds
+    def initialize(access_key_id: nil, secret_access_key: nil,
+                   region: nil, session_token: nil,
+                   host: nil, port: 443, ssl: true, timeout: 60,
+                   **)
+      region ||= "us-east-1"
+      @access_key_id = access_key_id
+      @secret_access_key = secret_access_key
+      @aws_region = region
+      @session_token = session_token
+      host ||= HOST_PATTERN % region
+      @aws_host = host
+      super(key: @access_key_id, host:, port:, ssl:, timeout:, persistent: false)
+    end
+
+    ##
+    # @return [Symbol] Returns the provider's name
+    def name
+      :bedrock
+    end
+
+    ##
+    # Provides an interface to the Bedrock Converse API
+    #
+    # @see https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
+    #
+    # @param prompt (see LLM::Provider#complete)
+    # @param params (see LLM::Provider#complete)
+    # @return (see LLM::Provider#complete)
+    def complete(prompt, params = {})
+      params, stream, tools, role = normalize_complete_params(params)
+      req, messages, body = build_complete_request(prompt, params, role, stream:)
+      tracer.set_request_metadata(user_input: extract_user_input(messages, fallback: prompt))
+      sign!(req, body)
+      model_id = model_id_for(req.path)
+      res, span, tracer = execute(request: req, stream:, operation: "chat", stream_parser:, model: model_id)
+      res = ResponseAdapter.adapt(res, type: :completion)
+        .extend(Module.new { define_method(:__tools__) { tools } })
+      tracer.on_request_finish(operation: "chat", model: model_id, res:, span:)
+      res
+    end
+
+    ##
+    # Provides an interface to Bedrock's ListFoundationModels API.
+    #
+    # @note
+    #  Unlike the Converse API (bedrock-runtime), this endpoint lives
+    #  on the control plane (bedrock.<region>.amazonaws.com).
+    #
+    # @see https://docs.aws.amazon.com/bedrock/latest/APIReference/API_ListFoundationModels.html
+    # @return [LLM::Bedrock::Models]
+    def models
+      LLM::Bedrock::Models.new(self)
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def files
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def images
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def audio
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def moderations
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def responses
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def vector_stores
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def embed(input, model: nil, **params)
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [String]
+    def assistant_role
+      "assistant"
+    end
+
+    ##
+    # Bedrock expects tool results as user messages containing
+    # `toolResult` content blocks rather than a distinct `tool` role.
+    # @return (see LLM::Provider#tool_role)
+    def tool_role
+      :user
+    end
+
+    ##
+    # @return [String]
+    def default_model
+      "deepseek.v3.2"
+    end
+
+    private
+
+    def headers
+      lock do
+        (@headers || {}).merge("Content-Type" => "application/json")
+      end
+    end
+
+    def credentials
+      LLM::Object.from(
+        access_key_id: @access_key_id,
+        secret_access_key: @secret_access_key,
+        aws_region: @aws_region,
+        host: @aws_host,
+        session_token: @session_token
+      )
+    end
+
+    def stream_parser
+      LLM::Bedrock::StreamParser
+    end
+
+    def error_handler
+      LLM::Bedrock::ErrorHandler
+    end
+
+    def stream_decoder
+      LLM::Bedrock::StreamDecoder
+    end
+
+    def normalize_complete_params(params)
+      params = {role: :user, model: default_model, max_tokens: 2048}.merge!(params)
+      tools = resolve_tools(params.delete(:tools))
+      params = [params, adapt_schema(params), adapt_tools(tools)].inject({}, &:merge!).compact
+      role, stream = params.delete(:role), params.delete(:stream)
+      params[:stream] = true if streamable?(stream) || stream == true
+      [params, stream, tools, role]
+    end
+
+    def build_complete_request(prompt, params, role, stream: nil)
+      messages = build_complete_messages(prompt, params, role)
+      model_id = params.delete(:model) || default_model
+      payload = build_converse_payload(messages, params)
+      body = LLM.json.dump(payload)
+      path = stream ? "/model/#{model_id}/converse-stream" \
+                    : "/model/#{model_id}/converse"
+      req = Net::HTTP::Post.new(path, headers)
+      set_body_stream(req, StringIO.new(body))
+      [req, messages, body]
+    end
+
+    def build_complete_messages(prompt, params, role)
+      if LLM::Prompt === prompt
+        [*(params.delete(:messages) || []), *prompt]
+      else
+        [*(params.delete(:messages) || []), Message.new(role, prompt)]
+      end
+    end
+
+    def build_converse_payload(messages, params)
+      adapted = adapt(messages)
+      payload = {}
+      payload[:system] = adapted[:system] if adapted[:system]&.any?
+      payload[:messages] = adapted[:messages]
+      inference_config = {}
+      inference_config[:maxTokens] = params.delete(:max_tokens) if params[:max_tokens]
+      inference_config[:temperature] = params.delete(:temperature) if params.key?(:temperature)
+      inference_config[:topP] = params.delete(:top_p) if params.key?(:top_p)
+      inference_config[:stopSequences] = params.delete(:stop) if params[:stop]
+      payload[:inferenceConfig] = inference_config unless inference_config.empty?
+      payload[:toolConfig] = params.delete(:toolConfig) if params[:toolConfig]
+      payload[:outputConfig] = params.delete(:outputConfig) if params[:outputConfig]
+      additional = {}
+      top_k = params.delete(:top_k)
+      additional[:top_k] = top_k if top_k
+      payload[:additionalModelRequestFields] = additional unless additional.empty?
+      payload
+    end
+
+    def extract_user_input(messages, fallback:)
+      message = messages.reverse.find(&:user?) || messages.last
+      value = message&.content || fallback
+      value.is_a?(String) ? value : LLM.json.dump(value)
+    end
+
+    def model_id_for(path)
+      path[%r{\A/model/(.+?)/converse(?:-stream)?\z}, 1] || default_model
+    end
+
+    def sign!(req, body)
+      Signature.new(
+        credentials:,
+        method: req.method,
+        path: req.path,
+        body:
+      ).sign!(req)
+    end
+  end
+end

data/lib/llm/stream/queue.rb

@@ -46,7 +46,7 @@ class LLM::Stream
     #   to wait on:
     #   - `:thread`: Use threads
     #   - `:task`: Use async tasks (requires async gem)
-    #   - `:fiber`: Use raw fibers
+    #   - `:fiber`: Use scheduler-backed fibers (requires Fiber.scheduler)
     #   - `:ractor`: Use Ruby ractors (class-based tools only; MCP tools are not supported)
     #   - `[:thread, :ractor]`: Wait for any queued thread or ractor work, in the
     #     given order. This is useful when different tools were spawned with

data/lib/llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LLM
-  VERSION = "7.0.0"
+  VERSION = "8.1.0"
 end

data/lib/llm.rb

@@ -20,6 +20,7 @@ module LLM
   require_relative "llm/mime"
   require_relative "llm/multipart"
   require_relative "llm/file"
+  require_relative "llm/pipe"
   require_relative "llm/stream"
   require_relative "llm/provider"
   require_relative "llm/context"
@@ -48,7 +49,24 @@ module LLM
 
   ##
   # @api private
-  def self.clients = @clients
+  def self.clients
+    @clients
+  end
+
+  ##
+  # Requires an optional runtime dependency
+  # @raise [LLM::DependencyError]
+  #  When the dependency cannot be loaded
+  def self.require(name)
+    super
+  rescue ::LoadError
+    names = {"xchan" => "xchan.rb", "net/http/persistent" => "net-http-persistent"}
+    name = names[name] || name
+    raise LLM::LoadError,
+      "#{name} is an optional runtime dependency but it does not appear to be installed. " \
+      "Consider 'gem install #{name}', adding '#{name}' to your Gemfile or " \
+      "opting out of the functionality provided by '#{name}'"
+  end
 
   ##
   # @param [Symbol, LLM::Provider] llm
@@ -143,6 +161,14 @@ module LLM
     LLM::OpenAI.new(**)
   end
 
+  ##
+  # @param (see LLM::Bedrock#initialize)
+  # @return (see LLM::Bedrock#initialize)
+  def bedrock(**)
+    lock(:require) { require_relative "llm/providers/bedrock" unless defined?(LLM::Bedrock) }
+    LLM::Bedrock.new(**)
+  end
+
   ##
   # @param key (see LLM::XAI#initialize)
   # @param host (see LLM::XAI#initialize)

data/llm.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   DESCRIPTION
 
   spec.license = "0BSD"
-  spec.required_ruby_version = ">= 3.2.0"
+  spec.required_ruby_version = ">= 3.3.0"
 
   spec.homepage = "https://github.com/llmrb/llm.rb"
   spec.metadata["homepage_uri"] = "https://github.com/llmrb/llm.rb"
@@ -57,5 +57,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "activerecord", "~> 8.0"
   spec.add_development_dependency "sequel", "~> 5.0"
   spec.add_development_dependency "sqlite3", "~> 2.0"
+  spec.add_development_dependency "xchan.rb", "~> 0.20"
   spec.add_development_dependency "pg", "~> 1.5"
 end