llm.rb 8.0.0 → 8.1.0

data/lib/llm/providers/bedrock/stream_parser.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+class LLM::Bedrock
+  ##
+  # Parses Bedrock Converse Stream events into a response body
+  # and emits stream callbacks (on_content, on_tool_call, etc.).
+  #
+  # Receives decoded JSON payloads from {StreamDecoder} along with
+  # the AWS Event Stream event type header.
+  #
+  # Bedrock Converse Stream event types:
+  #   messageStart — initial role
+  #   contentBlockStart — tool use or reasoning start
+  #   contentBlockDelta — text delta, tool input JSON, or reasoning text
+  #   contentBlockStop — content block finished
+  #   messageStop — final stop reason, usage metadata
+  #
+  # @api private
+  class StreamParser
+    TOOL_MARKER = "<｜DSML｜function_calls"
+
+    ##
+    # @return [Hash] Fully constructed response body
+    attr_reader :body
+
+    ##
+    # @param [#<<, LLM::Stream] stream
+    def initialize(stream)
+      @body = {"output" => {"message" => {"role" => "assistant", "content" => []}}}
+      @stream = stream
+      @text_markers = {}
+      @can_emit_content = stream.respond_to?(:on_content)
+      @can_emit_reasoning_content = stream.respond_to?(:on_reasoning_content)
+      @can_emit_tool_call = stream.respond_to?(:on_tool_call)
+      @can_push_content = stream.respond_to?(:<<)
+    end
+
+    ##
+    # @param [Hash] payload Decoded JSON from an event stream frame
+    # @param [String, nil] event_type The :event-type header value
+    # @return [self]
+    def parse!(payload, event_type: nil)
+      type = event_type || payload["type"]
+      case type
+      when "messageStart"
+        # { "role" => "assistant" }
+      when "contentBlockStart"
+        # { "contentBlockIndex" => 0, "start" => { "toolUse" => {...} } }
+        handle_content_block_start(payload)
+      when "contentBlockDelta"
+        # { "contentBlockIndex" => 0, "delta" => { "text" => "..." } }
+        handle_content_block_delta(payload)
+      when "contentBlockStop"
+        handle_content_block_stop(payload)
+      when "messageStop"
+        # { "stopReason" => "end_turn", "metadata" => {"usage" => {...}} }
+        merge_metadata(payload)
+      when "metadata"
+        # { "usage" => {...} }
+        merge_metadata(payload)
+      end
+      self
+    end
+
+    ##
+    # @return [void]
+    def free
+      @text_markers.clear
+    end
+
+    private
+
+    def handle_content_block_start(payload)
+      index = payload["contentBlockIndex"]
+      start_data = payload["start"] || {}
+      if (tool_use = start_data["toolUse"])
+        content[index] = {"toolUse" => {"toolUseId" => tool_use["toolUseId"], "name" => tool_use["name"], "input" => +""}}
+      elsif (reasoning = start_data["reasoningContent"])
+        content[index] = {"reasoningContent" => {"text" => +"", "signature" => reasoning["signature"]}.compact}
+      end
+    end
+
+    def handle_content_block_delta(payload)
+      index = payload["contentBlockIndex"]
+      delta = payload["delta"] || {}
+      if (text = delta["text"])
+        ensure_content_block(index, "text")
+        visible = filtered_text(index, text)
+        return if visible.empty?
+        content[index]["text"] ||= +""
+        content[index]["text"] << visible
+        emit_content(visible)
+      elsif (tool_input = delta.dig("toolUse", "input"))
+        ensure_content_block(index, "tool_use")
+        content[index]["toolUse"]["input"] ||= +""
+        content[index]["toolUse"]["input"] << tool_input
+      elsif (reasoning = delta["reasoningContent"])
+        ensure_content_block(index, "reasoning")
+        if reasoning["text"]
+          content[index]["reasoningContent"]["text"] ||= +""
+          content[index]["reasoningContent"]["text"] << reasoning["text"]
+          emit_reasoning_content(reasoning["text"])
+        end
+        if reasoning["signature"]
+          content[index]["reasoningContent"]["signature"] = reasoning["signature"]
+        end
+      end
+    end
+
+    def handle_content_block_stop(payload)
+      index = payload["contentBlockIndex"]
+      item = content[index]
+      return unless item
+      flush_text(index, item)
+      if item["toolUse"] && item["toolUse"]["input"].is_a?(String)
+        parsed = LLM.json.load(item["toolUse"]["input"])
+        item["toolUse"]["input"] = parsed.is_a?(Hash) ? parsed : {}
+        emit_tool(item)
+      end
+    rescue *LLM.json.parser_error
+      item["toolUse"]["input"] = {} if item&.dig("toolUse")
+    end
+
+    def ensure_content_block(index, type)
+      content[index] ||= case type
+      when "tool_use" then {"toolUse" => {"input" => +""}}
+      when "reasoning" then {"reasoningContent" => {"text" => +""}}
+      else {}
+      end
+    end
+
+    def filtered_text(index, text)
+      state = (@text_markers[index] ||= +"")
+      value = state << text
+      value.gsub!(TOOL_MARKER, "")
+      keep = marker_prefix_length(value)
+      @text_markers[index] = keep.zero? ? +"" : value[-keep..]
+      keep.zero? ? value : value[0...-keep]
+    end
+
+    def flush_text(index, item)
+      value = @text_markers.delete(index).to_s
+      return unless item["text"]
+      if value.empty?
+        content[index] = {} if item["text"].empty?
+      else
+        item["text"] << value
+        emit_content(value)
+      end
+    end
+
+    def marker_prefix_length(value)
+      [value.length, TOOL_MARKER.length - 1].min.downto(1) do |length|
+        return length if TOOL_MARKER.start_with?(value[-length..])
+      end
+      0
+    end
+
+    def merge_metadata(payload)
+      metadata = payload["metadata"] || payload
+      return unless metadata.is_a?(Hash)
+      usage = metadata["usage"]
+      @body["usage"] = usage if usage
+      @body["stopReason"] = payload["stopReason"] if payload["stopReason"]
+    end
+
+    def emit_content(value)
+      if @can_emit_content
+        @stream.on_content(value)
+      elsif @can_push_content
+        @stream << value
+      end
+    end
+
+    def emit_reasoning_content(value)
+      @stream.on_reasoning_content(value) if @can_emit_reasoning_content
+    end
+
+    def emit_tool(tool)
+      return unless @can_emit_tool_call
+      function, error = resolve_tool(tool)
+      @stream.on_tool_call(function, error)
+    end
+
+    def resolve_tool(tool)
+      payload = tool["toolUse"] || {}
+      registered = @stream.find_tool(payload["name"])
+      fn = (registered || LLM::Function.new(payload["name"])).dup.tap do |f|
+        f.id = payload["toolUseId"]
+        f.arguments = payload["input"] || {}
+        f.tracer = @stream.extra[:tracer]
+        f.model = @stream.extra[:model]
+      end
+      [fn, registered ? nil : @stream.tool_not_found(fn)]
+    end
+
+    def content
+      @body["output"]["message"]["content"]
+    end
+  end
+end

data/lib/llm/providers/bedrock.rb

@@ -0,0 +1,272 @@
+# frozen_string_literal: true
+
+module LLM
+  ##
+  # The Bedrock class implements a provider for
+  # [Amazon Bedrock](https://aws.amazon.com/bedrock/).
+  #
+  # Bedrock provides access to foundation models from Anthropic, Meta,
+  # Mistral, AI21 Labs, Cohere, and more through the AWS infrastructure.
+  # This provider uses the Bedrock Converse API for chat completions,
+  # and the Converse Stream API for streaming.
+  #
+  # Unlike other llm.rb providers which use API key authentication,
+  # Bedrock uses AWS Signature V4 (SigV4) for request signing.
+  # You must provide AWS credentials (access key, secret key, and region)
+  # instead of a single API key.
+  #
+  # Streaming uses the AWS Event Stream binary protocol instead of
+  # standard SSE. The binary framing is decoded inline using only
+  # Ruby's stdlib.
+  #
+  # @example
+  #   require "llm"
+  #
+  #   llm = LLM.bedrock(
+  #     access_key_id: ENV["AWS_ACCESS_KEY_ID"],
+  #     secret_access_key: ENV["AWS_SECRET_ACCESS_KEY"],
+  #     region: "us-east-1"
+  #   )
+  #   ctx = LLM::Context.new(llm)
+  #   ctx.talk "Hello, how are you?"
+  #   ctx.messages.select(&:assistant?).each { puts _1.content }
+  class Bedrock < Provider
+    require_relative "bedrock/signature"
+    require_relative "bedrock/error_handler"
+    require_relative "bedrock/request_adapter"
+    require_relative "bedrock/request_adapter/completion"
+    require_relative "bedrock/response_adapter"
+    require_relative "bedrock/response_adapter/completion"
+    require_relative "bedrock/response_adapter/models"
+    require_relative "bedrock/stream_decoder"
+    require_relative "bedrock/stream_parser"
+    require_relative "bedrock/models"
+
+    include RequestAdapter
+
+    HOST_PATTERN = "bedrock-runtime.%s.amazonaws.com"
+
+    ##
+    # @param [String] access_key_id AWS access key ID
+    # @param [String] secret_access_key AWS secret access key
+    # @param [String] region AWS region (e.g. "us-east-1")
+    # @param [String, nil] session_token AWS session token for temporary credentials
+    # @param [String, nil] host Override the Bedrock API host
+    # @param [Integer] port Connection port
+    # @param [Boolean] ssl Whether to use SSL
+    # @param [Integer] timeout Request timeout in seconds
+    def initialize(access_key_id: nil, secret_access_key: nil,
+                   region: nil, session_token: nil,
+                   host: nil, port: 443, ssl: true, timeout: 60,
+                   **)
+      region ||= "us-east-1"
+      @access_key_id = access_key_id
+      @secret_access_key = secret_access_key
+      @aws_region = region
+      @session_token = session_token
+      host ||= HOST_PATTERN % region
+      @aws_host = host
+      super(key: @access_key_id, host:, port:, ssl:, timeout:, persistent: false)
+    end
+
+    ##
+    # @return [Symbol] Returns the provider's name
+    def name
+      :bedrock
+    end
+
+    ##
+    # Provides an interface to the Bedrock Converse API
+    #
+    # @see https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
+    #
+    # @param prompt (see LLM::Provider#complete)
+    # @param params (see LLM::Provider#complete)
+    # @return (see LLM::Provider#complete)
+    def complete(prompt, params = {})
+      params, stream, tools, role = normalize_complete_params(params)
+      req, messages, body = build_complete_request(prompt, params, role, stream:)
+      tracer.set_request_metadata(user_input: extract_user_input(messages, fallback: prompt))
+      sign!(req, body)
+      model_id = model_id_for(req.path)
+      res, span, tracer = execute(request: req, stream:, operation: "chat", stream_parser:, model: model_id)
+      res = ResponseAdapter.adapt(res, type: :completion)
+        .extend(Module.new { define_method(:__tools__) { tools } })
+      tracer.on_request_finish(operation: "chat", model: model_id, res:, span:)
+      res
+    end
+
+    ##
+    # Provides an interface to Bedrock's ListFoundationModels API.
+    #
+    # @note
+    #  Unlike the Converse API (bedrock-runtime), this endpoint lives
+    #  on the control plane (bedrock.<region>.amazonaws.com).
+    #
+    # @see https://docs.aws.amazon.com/bedrock/latest/APIReference/API_ListFoundationModels.html
+    # @return [LLM::Bedrock::Models]
+    def models
+      LLM::Bedrock::Models.new(self)
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def files
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def images
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def audio
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def moderations
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def responses
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def vector_stores
+      raise NotImplementedError
+    end
+
+    ##
+    # @raise [NotImplementedError]
+    def embed(input, model: nil, **params)
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [String]
+    def assistant_role
+      "assistant"
+    end
+
+    ##
+    # Bedrock expects tool results as user messages containing
+    # `toolResult` content blocks rather than a distinct `tool` role.
+    # @return (see LLM::Provider#tool_role)
+    def tool_role
+      :user
+    end
+
+    ##
+    # @return [String]
+    def default_model
+      "deepseek.v3.2"
+    end
+
+    private
+
+    def headers
+      lock do
+        (@headers || {}).merge("Content-Type" => "application/json")
+      end
+    end
+
+    def credentials
+      LLM::Object.from(
+        access_key_id: @access_key_id,
+        secret_access_key: @secret_access_key,
+        aws_region: @aws_region,
+        host: @aws_host,
+        session_token: @session_token
+      )
+    end
+
+    def stream_parser
+      LLM::Bedrock::StreamParser
+    end
+
+    def error_handler
+      LLM::Bedrock::ErrorHandler
+    end
+
+    def stream_decoder
+      LLM::Bedrock::StreamDecoder
+    end
+
+    def normalize_complete_params(params)
+      params = {role: :user, model: default_model, max_tokens: 2048}.merge!(params)
+      tools = resolve_tools(params.delete(:tools))
+      params = [params, adapt_schema(params), adapt_tools(tools)].inject({}, &:merge!).compact
+      role, stream = params.delete(:role), params.delete(:stream)
+      params[:stream] = true if streamable?(stream) || stream == true
+      [params, stream, tools, role]
+    end
+
+    def build_complete_request(prompt, params, role, stream: nil)
+      messages = build_complete_messages(prompt, params, role)
+      model_id = params.delete(:model) || default_model
+      payload = build_converse_payload(messages, params)
+      body = LLM.json.dump(payload)
+      path = stream ? "/model/#{model_id}/converse-stream" \
+                    : "/model/#{model_id}/converse"
+      req = Net::HTTP::Post.new(path, headers)
+      set_body_stream(req, StringIO.new(body))
+      [req, messages, body]
+    end
+
+    def build_complete_messages(prompt, params, role)
+      if LLM::Prompt === prompt
+        [*(params.delete(:messages) || []), *prompt]
+      else
+        [*(params.delete(:messages) || []), Message.new(role, prompt)]
+      end
+    end
+
+    def build_converse_payload(messages, params)
+      adapted = adapt(messages)
+      payload = {}
+      payload[:system] = adapted[:system] if adapted[:system]&.any?
+      payload[:messages] = adapted[:messages]
+      inference_config = {}
+      inference_config[:maxTokens] = params.delete(:max_tokens) if params[:max_tokens]
+      inference_config[:temperature] = params.delete(:temperature) if params.key?(:temperature)
+      inference_config[:topP] = params.delete(:top_p) if params.key?(:top_p)
+      inference_config[:stopSequences] = params.delete(:stop) if params[:stop]
+      payload[:inferenceConfig] = inference_config unless inference_config.empty?
+      payload[:toolConfig] = params.delete(:toolConfig) if params[:toolConfig]
+      payload[:outputConfig] = params.delete(:outputConfig) if params[:outputConfig]
+      additional = {}
+      top_k = params.delete(:top_k)
+      additional[:top_k] = top_k if top_k
+      payload[:additionalModelRequestFields] = additional unless additional.empty?
+      payload
+    end
+
+    def extract_user_input(messages, fallback:)
+      message = messages.reverse.find(&:user?) || messages.last
+      value = message&.content || fallback
+      value.is_a?(String) ? value : LLM.json.dump(value)
+    end
+
+    def model_id_for(path)
+      path[%r{\A/model/(.+?)/converse(?:-stream)?\z}, 1] || default_model
+    end
+
+    def sign!(req, body)
+      Signature.new(
+        credentials:,
+        method: req.method,
+        path: req.path,
+        body:
+      ).sign!(req)
+    end
+  end
+end

data/lib/llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LLM
-  VERSION = "8.0.0"
+  VERSION = "8.1.0"
 end

data/lib/llm.rb

@@ -161,6 +161,14 @@ module LLM
     LLM::OpenAI.new(**)
   end
 
+  ##
+  # @param (see LLM::Bedrock#initialize)
+  # @return (see LLM::Bedrock#initialize)
+  def bedrock(**)
+    lock(:require) { require_relative "llm/providers/bedrock" unless defined?(LLM::Bedrock) }
+    LLM::Bedrock.new(**)
+  end
+
   ##
   # @param key (see LLM::XAI#initialize)
   # @param host (see LLM::XAI#initialize)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm.rb
 version: !ruby/object:Gem::Version
-  version: 8.0.0
+  version: 8.1.0
 platform: ruby
 authors:
 - Antar Azri
@@ -287,6 +287,7 @@ files:
 - LICENSE
 - README.md
 - data/anthropic.json
+- data/bedrock.json
 - data/deepseek.json
 - data/google.json
 - data/openai.json
@@ -369,6 +370,17 @@ files:
 - lib/llm/providers/anthropic/response_adapter/web_search.rb
 - lib/llm/providers/anthropic/stream_parser.rb
 - lib/llm/providers/anthropic/utils.rb
+- lib/llm/providers/bedrock.rb
+- lib/llm/providers/bedrock/error_handler.rb
+- lib/llm/providers/bedrock/models.rb
+- lib/llm/providers/bedrock/request_adapter.rb
+- lib/llm/providers/bedrock/request_adapter/completion.rb
+- lib/llm/providers/bedrock/response_adapter.rb
+- lib/llm/providers/bedrock/response_adapter/completion.rb
+- lib/llm/providers/bedrock/response_adapter/models.rb
+- lib/llm/providers/bedrock/signature.rb
+- lib/llm/providers/bedrock/stream_decoder.rb
+- lib/llm/providers/bedrock/stream_parser.rb
 - lib/llm/providers/deepseek.rb
 - lib/llm/providers/deepseek/request_adapter.rb
 - lib/llm/providers/deepseek/request_adapter/completion.rb