ruby_llm 2.1.1 → 2.2.0

data/lib/ruby_llm/tools/provider_tools.rb

@@ -68,6 +68,12 @@ module RubyLLM
         entries + tools_with_options.map { |name, options| { name: name.to_sym, options: options.to_h } }
       end
 
+      # Normalizes what an Agent's provider_tools block returns: a Hash of
+      # aliases and their options, like the macro's keywords, or a list.
+      def entries(value)
+        value.is_a?(Hash) ? normalize([], value) : Array(value).compact
+      end
+
       def normalize_positional(tool)
         case tool
         when Symbol, String then { name: tool.to_sym, options: {} }
@@ -85,6 +91,23 @@ module RubyLLM
         hash.key?(:raw) ? hash.size == 1 : hash.size == 2 && hash.key?(:name) && hash.key?(:options)
       end
 
+      # Adds normalized entries to those a chat already holds. An alias
+      # keeps one entry, in its place, with the latest options. A raw tool
+      # and an MCP server, which may be enabled several times, are entered
+      # once per identical definition.
+      def merge(entries, additions)
+        additions.each_with_object(entries.dup) do |entry, merged|
+          index = merged.index { |existing| same_tool?(existing, entry) }
+          index ? merged[index] = entry : merged << entry
+        end
+      end
+
+      def same_tool?(existing, entry)
+        return existing == entry if entry.key?(:raw) || entry[:name] == :mcp
+
+        existing[:name] == entry[:name]
+      end
+
       # Resolves normalized entries against a protocol's alias table.
       def resolve(entries, aliases:, owner:)
         entries.each_with_object(Resolution.new) do |entry, resolution|

data/lib/ruby_llm/tools/search.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Tools
+    # The +tool_search+ tool a chat with deferred tools gives the model when
+    # its provider has no tool search of its own. The model searches the
+    # deferred tools by what it needs to do, or names them, and the chat
+    # sends the definitions of the tools it found with every request after
+    # that. See Chat#deferred_tools.
+    class Search < Tool # :nodoc:
+      DEFAULT_LIMIT = 5
+      MAX_LIMIT = 20
+      SELECT = 'select:'
+      SUMMARY_LENGTH = 200
+
+      inline_limit false
+
+      def self.tool_name
+        'tool_search'
+      end
+
+      def self.loaded_names(content)
+        names = JSON.parse(content.to_s)['loaded'] if content.to_s.start_with?('{')
+        names.is_a?(Array) ? names.map(&:to_s) : []
+      rescue JSON::ParserError
+        []
+      end
+
+      def initialize(tools)
+        super()
+        @tools = tools
+      end
+
+      def description
+        'Loads tools you can call in this conversation but whose definitions are not shown yet. Search by what ' \
+          'you need to do, in a few words, and the tools that match become callable from your next step on, for ' \
+          "the rest of the conversation. Pass #{SELECT}name,other_name to load tools by name. " \
+          "Tools you can load: #{@tools.call.keys.join(', ')}."
+      end
+
+      def parameters_schema
+        {
+          'type' => 'object',
+          'properties' => {
+            'query' => { 'type' => 'string',
+                         'description' => "A few words about the task, or #{SELECT} and tool names, comma-separated" },
+            'limit' => { 'type' => 'integer',
+                         'description' => "How many matching tools to load, #{MAX_LIMIT} at most. " \
+                                          "#{SELECT} loads every tool it names",
+                         'default' => DEFAULT_LIMIT, 'minimum' => 1, 'maximum' => MAX_LIMIT }
+          },
+          'required' => %w[query],
+          'additionalProperties' => false
+        }
+      end
+
+      def execute(query:, limit: DEFAULT_LIMIT)
+        tools = @tools.call
+        query = query.to_s
+        found = matches(tools, query, limit.to_i.clamp(1, MAX_LIMIT))
+        result = found.empty? ? nothing_found(tools, query) : loaded(found)
+        missing = query.start_with?(SELECT) ? requested(query) - result[:loaded] : []
+        missing.empty? ? result : result.merge(not_loaded: missing)
+      end
+
+      private
+
+      def nothing_found(tools, query)
+        { loaded: [], error: "No tools match #{query}. Tools you can load: #{tools.keys.join(', ')}" }
+      end
+
+      def loaded(found)
+        { loaded: found.map(&:first), tools: found.map { |name, tool| { name:, description: summary(tool) } } }
+      end
+
+      def matches(tools, query, limit)
+        return selected(tools, requested(query)) if query.start_with?(SELECT)
+
+        terms = words(query)
+        ranked = tools.each_with_index.filter_map do |(name, tool), index|
+          score = terms.sum { |term| score(name.to_s, tool, term) }
+          [-score, index, name] if score.positive?
+        end
+        ranked.sort.first(limit).map { |*, name| [name.to_s, tools[name]] }
+      end
+
+      def requested(query)
+        query.delete_prefix(SELECT).split(',').map(&:strip).reject(&:empty?).uniq
+      end
+
+      def selected(tools, names)
+        names.filter_map { |name| [name, tools[name.to_sym]] if tools.key?(name.to_sym) }
+      end
+
+      def score(name, tool, term)
+        return 4 if words(name).include?(term)
+        return 2 if name.downcase.include?(term)
+        return 1 if words("#{tool.description} #{parameter_names(tool)}").any? { |word| word.start_with?(term) }
+
+        0
+      end
+
+      def words(text)
+        text.to_s.downcase.scan(/[[:alnum:]]+/)
+      end
+
+      def parameter_names(tool)
+        properties = tool.parameters_schema.to_h.transform_keys(&:to_s)['properties']
+        properties.is_a?(Hash) ? properties.keys.join(' ') : ''
+      end
+
+      def summary(tool)
+        text = tool.description.to_s.strip
+        text.length > SUMMARY_LENGTH ? "#{text[0, SUMMARY_LENGTH - 3].rstrip}..." : text
+      end
+    end
+  end
+end

data/lib/ruby_llm/transcription/segment.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class Transcription
+    # A timed piece of a transcript: an entry of Transcription#segments or
+    # Transcription#words, or the TranscriptionChunk#segment a stream
+    # completed. Readers cover what providers share; #[] and #to_h reach
+    # the rest under the provider's own field names.
+    #
+    #   transcription.segments.each do |segment|
+    #     puts "#{segment.speaker} at #{segment.start}s: #{segment.text}"
+    #   end
+    #
+    class Segment
+      include Support::Inspectable
+
+      def self.wrap(segments) # :nodoc:
+        segments&.map { |segment| new(segment) }
+      end
+
+      def initialize(attributes) # :nodoc:
+        @attributes = attributes.to_h.transform_keys(&:to_s)
+      end
+
+      # The spoken text, or +nil+ when the provider reports none.
+      def text = self['text'] || self['word'] || self['transcript']
+
+      # The speaker label, or +nil+ on models that do not separate speakers.
+      def speaker = self['speaker'] || self['speaker_id']
+
+      # When the text starts, in seconds from the start of the audio.
+      def start = self['start']
+
+      # When the text ends, in seconds from the start of the audio.
+      def end = self['end']
+
+      # Returns the provider's field named +key+, a String or Symbol.
+      def [](key)
+        @attributes[key.to_s]
+      end
+
+      # Returns the provider's field named +key+ the way Hash#fetch does.
+      def fetch(key, ...)
+        @attributes.fetch(key.to_s, ...)
+      end
+
+      # Returns +true+ when the provider sent a field named +key+.
+      def key?(key) = @attributes.key?(key.to_s)
+      alias has_key? key?
+
+      # Returns the provider's fields as a Hash with String keys.
+      def to_h = @attributes.dup
+      alias to_hash to_h
+
+      # Returns +true+ for a Segment or Hash holding the same fields.
+      def ==(other)
+        other.respond_to?(:to_h) && to_h == other.to_h.transform_keys(&:to_s)
+      end
+
+      private
+
+      def inspect_attributes
+        { text:, speaker:, start:, end: self.end }
+      end
+    end
+  end
+end

data/lib/ruby_llm/transcription/wav_audio.rb

@@ -40,7 +40,8 @@ module RubyLLM
 
           parse_format(body) if name == 'fmt '
           @data = body.b if name == 'data'
-          offset += 8 + length + (length % 2)
+          offset += 8 + length
+          offset += length % 2 if offset < content.bytesize
         end
         raise ArgumentError, 'WAV file contains a truncated chunk or padding' unless offset == content.bytesize
       end

data/lib/ruby_llm/transcription.rb

@@ -12,6 +12,7 @@ module RubyLLM
   class Transcription
     include Support::Inspectable
     include Accounting::Usage::Result
+    include Accounting::Usage::ModelInfo
 
     # The transcribed text.
     attr_reader :text
@@ -26,13 +27,14 @@ module RubyLLM
     # report it.
     attr_reader :duration
 
-    # The timed segments of the transcript as an array of hashes, or +nil+
-    # when the provider does not return segments. Diarization models add a
-    # speaker label to each segment.
+    # The timed segments of the transcript as an array of Segment objects,
+    # or +nil+ when the provider does not return segments. Diarization
+    # models add a speaker label to each segment.
     attr_reader :segments
 
-    # Word timing and speaker labels as an array of hashes, or +nil+ when
-    # the provider does not return them. Request timing with +timestamps:+.
+    # Word timing and speaker labels as an array of Segment objects, or
+    # +nil+ when the provider does not return them. Request timing with
+    # +timestamps:+.
     attr_reader :words
 
     def initialize(text:, model:, **attributes) # :nodoc:
@@ -40,10 +42,11 @@ module RubyLLM
       @model = model
       @language = attributes[:language]
       @duration = attributes[:duration]
-      @segments = attributes[:segments]
-      @words = attributes[:words]
+      @segments = Segment.wrap(attributes[:segments])
+      @words = Segment.wrap(attributes[:words])
       @input_tokens = attributes[:input_tokens]
       @output_tokens = attributes[:output_tokens]
+      @audio_seconds = attributes[:audio_seconds]
       @reported_cost = attributes[:reported_cost]
     end
 
@@ -51,7 +54,8 @@ module RubyLLM
     def tokens
       return ruby_llm_usage_tokens unless ruby_llm_usage_entries.empty?
 
-      Tokens.new(input: @input_tokens, output: @output_tokens, reported_cost: @reported_cost)
+      Tokens.new(input: @input_tokens, output: @output_tokens, audio_seconds: @audio_seconds,
+                 reported_cost: @reported_cost)
     end
 
     # Returns the transcription cost across every provider attempt.
@@ -61,16 +65,8 @@ module RubyLLM
       Cost.new(tokens:, model: model_info, category: :audio_tokens)
     end
 
-    attr_writer :model_info # :nodoc:
-
-    def model_info # :nodoc:
-      @model_info ||= RubyLLM.models.find(model)
-    rescue ModelNotFoundError
-      nil
-    end
-
     # Transcribes +audio_file+ and returns a Transcription. The file may be
-    # a path, URL, or IO object. Uses
+    # a path, URL, IO object, or Attachment. Uses
     # <tt>config.default_transcription_model</tt> unless +model:+ is given.
     # Pass +provider:+ and <tt>assume_model_exists: true</tt> to use a model
     # that is not in the registry.
@@ -85,21 +81,23 @@ module RubyLLM
     #   )
     #
     # +language:+ hints at the spoken language using the provider's accepted
-    # ISO 639-1 or BCP-47 language code.
+    # ISO 639-1 or BCP-47 language code. Some providers also take an Array
+    # of candidate languages and report the one spoken as #language.
     # +prompt:+ gives the model vocabulary or formatting guidance, and
     # +temperature:+ adjusts sampling. +format:+ selects the transcript
     # format in the provider's own vocabulary: OpenAI takes values such as
     # <tt>"text"</tt>, <tt>"verbose_json"</tt>, or <tt>"diarized_json"</tt>,
     # while Gemini takes a MIME type such as <tt>"text/plain"</tt>.
     # +speaker_names:+ and +speaker_references:+ label the speakers on
-    # models that support diarization; references may be paths, URLs, or IO
-    # objects. Option support depends on the selected model and provider.
+    # models that support diarization; references may be paths, URLs, IO
+    # objects, or Attachments. Option support depends on the selected model
+    # and provider.
     # +timestamps:+ requests +:word+ timestamps. Some providers also accept
     # +:segment+ or +:character+; unsupported granularities raise ArgumentError.
     # +provider_options:+ takes options in the provider's request vocabulary
     # and merges them into the rendered request as-is.
     # +owner:+ attributes the usage to a record, such as a user, and wins
-    # over RubyLLM.with_usage_owner.
+    # over RubyLLM.with_owner.
     #
     # Given a block, the transcript streams: each TranscriptionChunk is
     # yielded as it arrives and the completed Transcription is still
@@ -130,34 +128,12 @@ module RubyLLM
                         metadata: nil,
                         owner: nil,
                         &block)
-      config = context&.config || RubyLLM.config
-      model ||= config.default_transcription_model
-      model, provider_instance = Models.resolve(model, provider: provider, assume_model_exists: assume_model_exists,
-                                                       config: config)
-      empty_tokens = Tokens.new
-      payload = {
-        provider: provider_instance.slug,
-        provider_class: provider_instance.name,
-        model: model.id,
-        model_info: model,
-        language: language,
-        provider_options: provider_options,
-        metadata: metadata,
-        tokens: empty_tokens,
-        cost: Cost.new(tokens: empty_tokens, model:, category: :audio_tokens)
-      }
-
-      RubyLLM.instrument('transcription.ruby_llm', payload, config: config) do |event|
-        result = Accounting::Usage.owned_by(owner) do
-          provider_instance.transcribe(audio_file, model:, language:, format:, timestamps:, speaker_names:,
-                                                   speaker_references:, provider_options:, prompt:,
-                                                   temperature:, &block)
-        end
-        event[:result] = result
-        event[:response_model] = result.model
-        event[:tokens] = result.tokens
-        event[:cost] = result.cost
-        result
+      Accounting::Operation.new(
+        'transcription', model, default: :default_transcription_model, category: :audio_tokens, provider:,
+                                assume_model_exists:, context:, owner:
+      ).run(streaming: !block.nil?, language:, temperature:, provider_options:, metadata:) do |provider_instance, model|
+        provider_instance.transcribe(audio_file, model:, language:, format:, timestamps:, speaker_names:,
+                                                 speaker_references:, provider_options:, prompt:, temperature:, &block)
       end
     end
 

data/lib/ruby_llm/transcription_chunk.rb

@@ -13,19 +13,19 @@ module RubyLLM
     include Support::Inspectable
 
     # Text deltas, arriving as the model transcribes.
-    DELTA = 'transcript.text.delta'
+    DELTA = :delta
 
     # A tentative transcript that may change before its segment completes.
-    PARTIAL = 'transcript.text.partial'
+    PARTIAL = :partial
 
     # A completed segment, on models that return timed or diarized segments.
-    SEGMENT = 'transcript.text.segment'
+    SEGMENT = :segment
 
     # The final event, carrying the complete transcript.
-    DONE = 'transcript.text.done'
+    DONE = :done
 
-    # The normalized event type, such as
-    # <tt>"transcript.text.delta"</tt>.
+    # The event type: +:delta+, +:partial+, +:segment+, or +:done+, or +nil+
+    # for a provider event RubyLLM does not normalize.
     attr_reader :type
 
     # The text added by this event, or +nil+ for events that add no text.
@@ -34,8 +34,8 @@ module RubyLLM
     # The transcript for a partial event or the complete final transcript.
     attr_reader :text
 
-    # The segment this event completed as a Hash, or +nil+. Diarization
-    # models label each segment with a speaker.
+    # The Transcription::Segment this event completed, or +nil+.
+    # Diarization models label each segment with a speaker.
     attr_reader :segment
 
     # The parsed provider event, for fields RubyLLM does not normalize.
@@ -45,7 +45,7 @@ module RubyLLM
       @type = type
       @delta = delta
       @text = text
-      @segment = segment
+      @segment = Transcription::Segment.new(segment) if segment
       @raw = raw
     end