ruby_llm-voyage 0.1.0

data/lib/ruby_llm/providers/voyage/contextualized_embeddings.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Voyage
+      # Voyage contextualized chunk embedding mapping.
+      # @api private
+      module ContextualizedEmbeddings
+        # Executes a contextualized chunk embedding request.
+        # @return [RubyLLM::Voyage::ContextualizedEmbedding]
+        def contextualized_embed(inputs, model:, input_type: RubyLLM::Voyage::UNSET, dimensions: nil,
+                                 output_dtype: RubyLLM::Voyage::UNSET,
+                                 encoding_format: RubyLLM::Voyage::UNSET, auto_chunk: nil, chunk_size: nil,
+                                 chunk_overlap: nil, provider_options: {})
+          payload = {
+            inputs: inputs,
+            model: model,
+            input_type: contextualized_option(input_type, :voyage_default_input_type),
+            output_dimension: dimensions || @config.voyage_default_output_dimension,
+            output_dtype: contextualized_option(output_dtype, :voyage_default_output_dtype),
+            encoding_format: contextualized_option(encoding_format, :voyage_default_encoding_format),
+            enable_auto_chunking: auto_chunk,
+            chunk_size: chunk_size,
+            chunk_overlap: chunk_overlap
+          }.compact.merge(provider_options.transform_keys(&:to_sym))
+          response = @connection.post('contextualizedembeddings', payload)
+          parse_contextualized_response(response, output_dtype: payload[:output_dtype])
+        end
+
+        private
+
+        def contextualized_option(value, config_key)
+          value = @config.public_send(config_key) if value.equal?(RubyLLM::Voyage::UNSET)
+          value&.to_s
+        end
+
+        def parse_contextualized_response(response, output_dtype: nil)
+          body = response.body
+          results = body.fetch('data').map do |group|
+            chunks = group.fetch('data').map do |item|
+              vector = item.fetch('embedding')
+              vector = RubyLLM::Voyage::VectorDecoder.decode(vector, output_dtype) if vector.is_a?(String)
+              RubyLLM::Voyage::ContextualizedChunk.new(
+                embedding: vector,
+                index: item.fetch('index'),
+                text: item['text']
+              )
+            end
+            RubyLLM::Voyage::ContextualizedResult.new(chunks: chunks, index: group.fetch('index'))
+          end
+
+          RubyLLM::Voyage::ContextualizedEmbedding.new(
+            results: results,
+            model: body['model'],
+            input_tokens: body.dig('usage', 'total_tokens') || 0,
+            chunker_version: body['chunker_version']
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/voyage/embeddings.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Voyage
+      # Voyage embeddings request and response mapping.
+      # @api private
+      module Embeddings
+        # Returns the embeddings endpoint relative to the configured API base.
+        # @return [String]
+        # @api private
+        def embedding_url(...)
+          'embeddings'
+        end
+
+        # Converts RubyLLM embedding arguments to a Voyage request body.
+        # A +nil+ dimensions argument falls back to the configured
+        # +voyage_default_output_dimension+.
+        # @param text [String, Array<String>]
+        # @param model [String]
+        # @param dimensions [Integer, nil]
+        # @param input_type [String, Symbol, nil]
+        # @param truncation [Boolean, nil]
+        # @param output_dtype [String, Symbol, nil]
+        # @param encoding_format [String, Symbol, nil]
+        # @return [Hash]
+        # @api private
+        def render_embedding_payload(text, model:, dimensions: nil, input_type: RubyLLM::Voyage::UNSET,
+                                     truncation: RubyLLM::Voyage::UNSET, output_dtype: RubyLLM::Voyage::UNSET,
+                                     encoding_format: RubyLLM::Voyage::UNSET)
+          {
+            model: model,
+            input: text,
+            input_type: enum_option(input_type, :voyage_default_input_type),
+            truncation: configured_option(truncation, :voyage_default_truncation),
+            output_dimension: dimensions || @config.voyage_default_output_dimension,
+            output_dtype: enum_option(output_dtype, :voyage_default_output_dtype),
+            encoding_format: enum_option(encoding_format, :voyage_default_encoding_format)
+          }.compact
+        end
+
+        # Converts a Voyage response into a RubyLLM embedding result.
+        # @param response [Faraday::Response]
+        # @param model [String]
+        # @param text [String, Array<String>]
+        # @param output_dtype [String, Symbol, nil]
+        # @return [RubyLLM::Embedding]
+        # @raise [KeyError] when the response omits embedding data
+        # @raise [ArgumentError] when Base64 data or its output type is invalid
+        # @api private
+        def parse_embedding_response(response, model:, text:, output_dtype: nil)
+          data = response.body
+          vectors = data.fetch('data').map { |item| item.fetch('embedding') }
+          if vectors.first.is_a?(String)
+            vectors.map! do |vector|
+              RubyLLM::Voyage::VectorDecoder.decode(vector, output_dtype)
+            end
+          end
+          vectors = vectors.first if vectors.length == 1 && !text.is_a?(Array)
+
+          Embedding.new(
+            vectors: vectors,
+            model: data['model'] || model,
+            input_tokens: data.dig('usage', 'total_tokens') || 0
+          )
+        end
+
+        private
+
+        def configured_option(value, config_key)
+          value.equal?(RubyLLM::Voyage::UNSET) ? @config.public_send(config_key) : value
+        end
+
+        def enum_option(value, config_key)
+          configured_option(value, config_key)&.to_s
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/voyage/files.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module RubyLLM
+  module Providers
+    class Voyage
+      # Voyage Files API operations.
+      # @api private
+      module Files
+        # Maximum redirect hops followed when downloading file content.
+        MAX_DOWNLOAD_REDIRECTS = 3
+        private_constant :MAX_DOWNLOAD_REDIRECTS
+
+        # Uploads one JSONL file.
+        # @param path [String, Pathname]
+        # @param purpose [String]
+        # @return [RubyLLM::Voyage::VoyageFile]
+        def upload_file(path, purpose: 'batch')
+          expanded = File.expand_path(path)
+          unless File.extname(expanded).casecmp?('.jsonl')
+            raise ArgumentError, 'Voyage file uploads must use the .jsonl extension'
+          end
+
+          # RubyLLM's connection stack loads faraday-multipart and uses this
+          # same class for its own uploads; 1.x has no public helper for it.
+          part = Faraday::Multipart::FilePart.new(expanded, 'application/jsonl', File.basename(expanded))
+          response = @connection.post('files', { file: part, purpose: purpose })
+          parse_file(response.body)
+        end
+
+        # Retrieves file metadata.
+        # @param file_id [String]
+        # @return [RubyLLM::Voyage::VoyageFile]
+        def retrieve_file(file_id)
+          parse_file(@connection.get("files/#{escape_id(file_id)}").body)
+        end
+
+        # Lists file metadata with cursor pagination.
+        # @return [RubyLLM::Voyage::Page]
+        def list_files(purpose: nil, limit: nil, order: nil, after: nil)
+          query = URI.encode_www_form({ purpose: purpose, limit: limit, order: order, after: after }.compact)
+          path = query.empty? ? 'files' : "files?#{query}"
+          parse_page(@connection.get(path).body) { |item| parse_file(item) }
+        end
+
+        # Downloads raw file content.
+        # @param file_id [String]
+        # @return [String]
+        def file_content(file_id)
+          response = @connection.get("files/#{escape_id(file_id)}/content") do |request|
+            request.headers['Accept'] = 'text/plain'
+          end
+          return response.body unless response.status.between?(300, 399)
+
+          download_redirect(response.headers['location'])
+        end
+
+        # Atomically deletes one or more files. Voyage treats the operation
+        # as all-or-nothing and reports the outcome in a +success+ field,
+        # with +error_file_ids+ available in +raw+ on failure.
+        # @param file_ids [Array<String>]
+        # @return [RubyLLM::Voyage::FileDeletion]
+        def delete_files(file_ids)
+          body = @connection.post('files/delete', { file_ids: file_ids }).body
+          deleted = body.is_a?(Hash) && body.key?('success') ? body['success'] : true
+          RubyLLM::Voyage::FileDeletion.new(file_ids: file_ids, deleted: deleted, raw: body)
+        end
+
+        private
+
+        def parse_file(item)
+          RubyLLM::Voyage::VoyageFile.new(
+            id: item['id'], object: item['object'], purpose: item['purpose'], filename: item['filename'],
+            bytes: item['bytes'], created_at: item['created_at'], expires_at: item['expires_at'], raw: item
+          )
+        end
+
+        def parse_page(body, &)
+          RubyLLM::Voyage::Page.new(
+            data: body.fetch('data').map(&),
+            first_id: body['first_id'], last_id: body['last_id'], has_more: body['has_more'], raw: body
+          )
+        end
+
+        def escape_id(id)
+          URI.encode_www_form_component(id.to_s)
+        end
+
+        def download_redirect(location)
+          connection = download_connection
+
+          MAX_DOWNLOAD_REDIRECTS.times do
+            uri = validated_redirect_uri(location)
+            response = connection.get(uri.to_s) { |request| request.headers['Accept'] = 'text/plain' }
+            return response.body unless response.status.between?(300, 399)
+
+            location = response.headers['location']
+          end
+
+          raise RubyLLM::Error, 'Voyage file download followed too many redirects'
+        end
+
+        def validated_redirect_uri(location)
+          uri = URI(location.to_s)
+          unless uri.is_a?(URI::HTTPS) && uri.host && !uri.userinfo
+            raise RubyLLM::Error, 'Voyage returned an unsafe file download redirect'
+          end
+
+          uri
+        end
+
+        # Signed storage URLs must be fetched outside the provider connection
+        # so the Voyage Authorization header is not forwarded. Uses RubyLLM's
+        # own side-connection builder and honors its configured adapter and
+        # timeout.
+        def download_connection
+          RubyLLM::Connection.basic do |faraday|
+            faraday.options.timeout = @config.request_timeout
+            faraday.adapter(@config.faraday_adapter || :net_http)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/voyage/reranking.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Voyage
+      # Voyage reranking request and response mapping.
+      # @api private
+      module Reranking
+        # Executes a Voyage reranking request.
+        # @return [RubyLLM::Voyage::Reranking]
+        def rerank(query:, documents:, model:, top_k: nil, return_documents: RubyLLM::Voyage::UNSET,
+                   truncation: RubyLLM::Voyage::UNSET, provider_options: {})
+          payload = {
+            query: query,
+            documents: documents,
+            model: model,
+            top_k: top_k,
+            return_documents: return_documents,
+            truncation: truncation
+          }.reject { |_key, value| value.nil? || value.equal?(RubyLLM::Voyage::UNSET) }
+           .merge(provider_options.transform_keys(&:to_sym))
+          response = @connection.post('rerank', payload)
+          parse_rerank_response(response)
+        end
+
+        private
+
+        def parse_rerank_response(response)
+          body = response.body
+          results = body.fetch('data').map do |item|
+            RubyLLM::Voyage::RerankResult.new(
+              index: item.fetch('index'),
+              relevance_score: item.fetch('relevance_score'),
+              document: item['document']
+            )
+          end
+
+          RubyLLM::Voyage::Reranking.new(
+            results: results,
+            model: body['model'],
+            input_tokens: body.dig('usage', 'total_tokens') || 0
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/voyage.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  # Provider implementations registered with RubyLLM.
+  module Providers
+    # Voyage retrieval API provider for RubyLLM.
+    #
+    # Most applications should use +RubyLLM.embed+ or
+    # {RubyLLM::Voyage.embed} instead of instantiating this class directly.
+    class Voyage < Provider
+      require 'ruby_llm/providers/voyage/embeddings'
+      require 'ruby_llm/providers/voyage/reranking'
+      require 'ruby_llm/providers/voyage/contextualized_embeddings'
+      require 'ruby_llm/providers/voyage/files'
+
+      include Voyage::Embeddings
+      include Voyage::Reranking
+      include Voyage::ContextualizedEmbeddings
+      include Voyage::Files
+
+      # Returns the configured Voyage API base URL.
+      # @return [String]
+      def api_base
+        @config.voyage_api_base || 'https://api.voyageai.com/v1'
+      end
+
+      # Returns authentication headers for Voyage requests.
+      # @return [Hash{String => String}]
+      def headers
+        { 'Authorization' => "Bearer #{@config.voyage_api_key}" }
+      end
+
+      # RubyLLM 1.x does not expose per-request provider options for embeddings,
+      # so the extension's facade calls this richer provider method directly.
+      #
+      # @api private
+      # @param text [String, Array<String>]
+      # @param model [String]
+      # @param dimensions [Integer, nil]
+      # @param provider_options [Hash]
+      # @return [RubyLLM::Embedding]
+      def embed(text, model:, dimensions: nil, provider_options: {}, **options)
+        payload = render_embedding_payload(text, model:, dimensions:, **options)
+                  .merge(provider_options.transform_keys(&:to_sym))
+        response = @connection.post(embedding_url(model:), payload)
+        parse_embedding_response(response, model:, text:, output_dtype: payload[:output_dtype])
+      end
+
+      class << self
+        # Lists configuration keys registered for the Voyage provider.
+        # @return [Array<Symbol>]
+        # @api private
+        def configuration_options
+          %i[
+            voyage_api_key
+            voyage_api_base
+            voyage_default_input_type
+            voyage_default_truncation
+            voyage_default_output_dimension
+            voyage_default_output_dtype
+            voyage_default_encoding_format
+          ]
+        end
+
+        # Lists configuration keys required to use the Voyage provider.
+        # @return [Array<Symbol>]
+        # @api private
+        def configuration_requirements
+          %i[voyage_api_key]
+        end
+
+        # Indicates that Voyage model IDs can be used without registry entries.
+        # @return [Boolean]
+        def assume_models_exist?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/voyage/results.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Voyage
+    # One document returned by a reranking request.
+    RerankResult = Struct.new(:index, :relevance_score, :document, keyword_init: true)
+
+    # Typed result returned by {Voyage.rerank}.
+    Reranking = Struct.new(:results, :model, :input_tokens, keyword_init: true) do
+      # Reorders the original candidate collection by relevance.
+      #
+      # Pass the same collection whose contents were sent as +documents:+.
+      # Returns the matching items in reranked order, truncated to +top_k+
+      # when one was requested.
+      #
+      # @param collection [Array, #to_a] candidates in their original order
+      # @return [Array] the reranked items
+      # @raise [IndexError] when the collection is smaller than the
+      #   document list that was reranked
+      #
+      # @example
+      #   reranking = RubyLLM::Voyage.rerank(
+      #     query: query, documents: articles.map(&:content), top_k: 5
+      #   )
+      #   reranking.reorder(articles) # => the five most relevant articles
+      def reorder(collection)
+        items = collection.to_a
+        results.map { |result| items.fetch(result.index) }
+      end
+    end
+
+    # One contextualized chunk and its vector.
+    ContextualizedChunk = Struct.new(:embedding, :index, :text, keyword_init: true) do
+      # Summarizes the vector so console output stays readable.
+      # @return [String]
+      def inspect
+        dimensions = embedding.respond_to?(:length) ? "[#{embedding.length} values]" : embedding.inspect
+        "#<RubyLLM::Voyage::ContextualizedChunk index=#{index} embedding=#{dimensions} " \
+          "text=#{text&.slice(0, 60).inspect}>"
+      end
+    end
+
+    # Contextualized chunks belonging to one input document or query.
+    ContextualizedResult = Struct.new(:chunks, :index, keyword_init: true)
+
+    # Typed result returned by {Voyage.contextualized_embed}.
+    ContextualizedEmbedding = Struct.new(
+      :results,
+      :model,
+      :input_tokens,
+      :chunker_version,
+      keyword_init: true
+    )
+
+    # Metadata for a file managed by Voyage's Files API.
+    VoyageFile = Struct.new(
+      :id,
+      :object,
+      :purpose,
+      :filename,
+      :bytes,
+      :created_at,
+      :expires_at,
+      :raw,
+      keyword_init: true
+    )
+
+    # A cursor-based page returned by a Voyage list endpoint.
+    Page = Struct.new(:data, :first_id, :last_id, :has_more, :raw, keyword_init: true)
+
+    # Result of an all-or-nothing bulk file deletion.
+    FileDeletion = Struct.new(:file_ids, :deleted, :raw, keyword_init: true)
+  end
+end

data/lib/ruby_llm/voyage/vector_decoder.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Voyage
+    # Decodes Voyage Base64 vector representations.
+    #
+    # The gem decodes API responses automatically; use this directly when
+    # parsing embeddings out of Voyage batch result files, which contain the
+    # same Base64 encoding.
+    module VectorDecoder
+      module_function
+
+      # Decodes a Base64 vector for the requested Voyage output type.
+      # @param vector [String] Base64-encoded vector data
+      # @param output_dtype [String, Symbol, nil] the +output_dtype+ the
+      #   embedding was requested with; +nil+ decodes as +float+
+      # @return [Array<Float>, Array<Integer>]
+      # @raise [ArgumentError] when the Base64 data or output type is invalid
+      #
+      # @example Decode an embedding from a batch results file
+      #   line = JSON.parse(jsonl_line)
+      #   encoded = line.dig('response', 'body', 'data', 0, 'embedding')
+      #   RubyLLM::Voyage::VectorDecoder.decode(encoded, :float)
+      def decode(vector, output_dtype)
+        # unpack1('m0') is strict Base64: it raises ArgumentError on invalid
+        # input, matching Base64.strict_decode64 without requiring the
+        # base64 gem (no longer a default gem as of Ruby 3.5).
+        bytes = vector.unpack1('m0')
+
+        case (output_dtype || 'float').to_s
+        when 'float' then bytes.unpack('e*')
+        when 'int8', 'binary' then bytes.unpack('c*')
+        when 'uint8', 'ubinary' then bytes.unpack('C*')
+        else raise ArgumentError, "Unsupported Voyage output dtype: #{output_dtype.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/voyage/version.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  # Voyage retrieval integration for RubyLLM.
+  #
+  # Use {.embed_query} and {.embed_documents} for the two sides of retrieval,
+  # {.embed} for request-level Voyage options, or the conventional
+  # +RubyLLM.embed(..., provider: :voyage)+ interface.
+  module Voyage
+    # Current gem version.
+    VERSION = '0.1.0'
+
+    # Internal sentinel used to distinguish an omitted option from an explicit
+    # +nil+ value.
+    # @api private
+    UNSET = Object.new.freeze
+  end
+end