ruby-gemini-api 1.0.0 → 1.2.0

data/lib/gemini/live/session.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require "json"
+require "base64"
+
+module Gemini
+  class Live
+    # Live API session manager
+    class Session
+      attr_reader :configuration, :last_resumption_token, :usage_metadata
+
+      def initialize(api_key:, configuration:)
+        @api_key = api_key
+        @configuration = configuration
+        @event_handlers = Hash.new { |h, k| h[k] = [] }
+        @connected = false
+        @setup_complete = false
+        @last_resumption_token = nil
+        @usage_metadata = nil
+        @connection = nil
+
+        setup_connection
+      end
+
+      # Register event handler
+      # Supported events:
+      #   :setup_complete - Session setup completed
+      #   :text           - Text response received (text)
+      #   :audio          - Audio data received (base64_data, mime_type)
+      #   :data           - Other inline data received (base64_data, mime_type)
+      #   :tool_call      - Tool call requested (function_calls)
+      #   :interrupted    - User interrupted the model
+      #   :turn_complete  - Model turn completed
+      #   :generation_complete - Generation completed
+      #   :usage_metadata - Token usage info received (metadata)
+      #   :session_resumption - Session resumption token updated (update)
+      #   :go_away        - Connection will close soon (info)
+      #   :error          - Error occurred (error)
+      #   :close          - Connection closed (code, reason)
+      def on(event, &block)
+        @event_handlers[event.to_sym] << block
+        self
+      end
+
+      # Send text message via clientContent.turns. This is the legacy form
+      # used by native-audio Live models. Newer models such as
+      # gemini-3.1-flash-live-preview reject this payload — use
+      # #send_realtime_text instead, which works on every Live model.
+      def send_text(text, turn_complete: true)
+        ensure_setup_complete!
+        message = MessageBuilder.client_content(
+          text: text,
+          turn_complete: turn_complete
+        )
+        @connection.send(message)
+      end
+
+      # Send text input via realtimeInput.text (universal form).
+      # Works with every currently-deployed Live model, including
+      # gemini-3.1-flash-live-preview and native-audio variants.
+      def send_realtime_text(text)
+        ensure_setup_complete!
+        @connection.send(MessageBuilder.realtime_text(text))
+      end
+
+      # Send audio data (Base64 encoded PCM)
+      def send_audio(audio_data, mime_type: "audio/pcm;rate=16000")
+        ensure_setup_complete!
+        encoded_data = audio_data.is_a?(String) && audio_data.encoding == Encoding::BINARY ?
+          Base64.strict_encode64(audio_data) : audio_data
+        message = MessageBuilder.realtime_input(
+          audio_data: encoded_data,
+          mime_type: mime_type
+        )
+        @connection.send(message)
+      end
+
+      # Send video/image data (Base64 encoded)
+      def send_video(image_data, mime_type: "image/jpeg")
+        ensure_setup_complete!
+        encoded_data = image_data.is_a?(String) && image_data.encoding == Encoding::BINARY ?
+          Base64.strict_encode64(image_data) : image_data
+        message = MessageBuilder.realtime_input(
+          video_data: encoded_data,
+          mime_type: mime_type
+        )
+        @connection.send(message)
+      end
+
+      # Send tool response
+      def send_tool_response(function_responses)
+        ensure_setup_complete!
+        message = MessageBuilder.tool_response(function_responses)
+        @connection.send(message)
+      end
+
+      # Manual VAD control - signal activity start
+      def activity_start
+        ensure_setup_complete!
+        @connection.send(MessageBuilder.activity_start)
+      end
+
+      # Manual VAD control - signal activity end
+      def activity_end
+        ensure_setup_complete!
+        @connection.send(MessageBuilder.activity_end)
+      end
+
+      # Close the session
+      def close
+        @connection&.close
+        @connected = false
+        @setup_complete = false
+      end
+
+      def connected?
+        @connected && @connection&.connected?
+      end
+
+      def setup_complete?
+        @setup_complete
+      end
+
+      private
+
+      def setup_connection
+        @connection = Connection.new(
+          api_key: @api_key,
+          on_message: method(:handle_message),
+          on_open: method(:handle_open),
+          on_error: method(:handle_error),
+          on_close: method(:handle_close)
+        )
+        @connection.connect
+        @connected = true
+      end
+
+      def handle_open
+        # Send setup message immediately after connection opens
+        setup_message = MessageBuilder.setup(@configuration)
+        @connection.send(setup_message)
+      end
+
+      def handle_message(data)
+        parsed = JSON.parse(data, symbolize_names: true)
+
+        if parsed[:setupComplete]
+          @setup_complete = true
+          emit(:setup_complete)
+        elsif parsed[:serverContent]
+          handle_server_content(parsed[:serverContent])
+        elsif parsed[:toolCall]
+          emit(:tool_call, parsed[:toolCall][:functionCalls])
+        elsif parsed[:usageMetadata]
+          @usage_metadata = parsed[:usageMetadata]
+          emit(:usage_metadata, parsed[:usageMetadata])
+        elsif parsed[:sessionResumptionUpdate]
+          handle_session_resumption(parsed[:sessionResumptionUpdate])
+        elsif parsed[:goAway]
+          emit(:go_away, parsed[:goAway])
+        end
+      rescue JSON::ParserError => e
+        emit(:error, e)
+      end
+
+      def handle_server_content(content)
+        # Check for interruption
+        if content[:interrupted]
+          emit(:interrupted)
+          return
+        end
+
+        # Check for generation complete
+        if content[:generationComplete]
+          emit(:generation_complete)
+        end
+
+        # Process model turn
+        model_turn = content[:modelTurn]
+        if model_turn
+          model_turn[:parts]&.each do |part|
+            if part[:text]
+              emit(:text, part[:text])
+            elsif part[:inlineData]
+              inline = part[:inlineData]
+              if inline[:mimeType]&.start_with?("audio/")
+                emit(:audio, inline[:data], inline[:mimeType])
+              else
+                emit(:data, inline[:data], inline[:mimeType])
+              end
+            end
+          end
+        end
+
+        # Check for turn complete
+        emit(:turn_complete) if content[:turnComplete]
+      end
+
+      def handle_session_resumption(update)
+        @last_resumption_token = update[:newHandle]
+        emit(:session_resumption, update)
+      end
+
+      def handle_error(error)
+        emit(:error, error)
+      end
+
+      def handle_close(code, reason)
+        @connected = false
+        @setup_complete = false
+        emit(:close, code, reason)
+      end
+
+      def emit(event, *args)
+        @event_handlers[event].each { |handler| handler.call(*args) }
+      end
+
+      def ensure_setup_complete!
+        raise Gemini::Error, "Session setup not complete. Wait for :setup_complete event." unless @setup_complete
+      end
+    end
+  end
+end

data/lib/gemini/live.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative "live/configuration"
+require_relative "live/message_builder"
+require_relative "live/connection"
+require_relative "live/session"
+
+module Gemini
+  # Live API client for real-time audio/video/text interactions
+  #
+  # @example Basic text conversation
+  #   client = Gemini::Client.new(api_key)
+  #   session = client.live.connect(model: "gemini-2.5-flash-live-preview")
+  #
+  #   session.on(:setup_complete) { puts "Connected!" }
+  #   session.on(:text) { |text| puts "AI: #{text}" }
+  #   session.on(:error) { |e| puts "Error: #{e}" }
+  #
+  #   session.send_text("Hello!")
+  #   sleep 5
+  #   session.close
+  #
+  # @example Audio conversation
+  #   session = client.live.connect(
+  #     model: "gemini-2.5-flash-live-preview",
+  #     response_modality: "AUDIO",
+  #     voice_name: "Puck"
+  #   )
+  #
+  #   session.on(:audio) { |data, mime| play_audio(data) }
+  #   session.send_audio(pcm_data)  # 16-bit PCM, 16kHz, mono
+  #
+  # @example With block (auto-close)
+  #   client.live.connect(model: "gemini-2.5-flash-live-preview") do |session|
+  #     session.on(:text) { |text| puts text }
+  #     session.send_text("Hello!")
+  #     sleep 5
+  #   end  # session.close called automatically
+  #
+  class Live
+    def initialize(client:)
+      @client = client
+    end
+
+    # Establish a WebSocket connection and return a session
+    #
+    # @param model [String] Model to use (default: "gemini-2.5-flash-live-preview")
+    # @param response_modality [String] "TEXT" or "AUDIO" (default: "TEXT")
+    # @param voice_name [String] Voice for audio responses (Puck, Charon, Kore, etc.)
+    # @param system_instruction [String] System prompt
+    # @param tools [Array] Tool definitions for function calling
+    # @param context_window_compression [Hash] Compression settings for long sessions
+    # @param session_resumption [Hash] Session resumption settings
+    # @param automatic_activity_detection [Boolean] Enable/disable automatic VAD (default: true)
+    # @param media_resolution [String] Media resolution setting
+    # @param output_audio_transcription [Boolean] Enable audio transcription (default: false)
+    # @yield [session] If block given, yields the session and closes it when block returns
+    # @return [Gemini::Live::Session] The live session
+    #
+    def connect(
+      model: Configuration::DEFAULT_MODEL,
+      response_modality: "TEXT",
+      voice_name: nil,
+      system_instruction: nil,
+      tools: nil,
+      context_window_compression: nil,
+      session_resumption: nil,
+      automatic_activity_detection: true,
+      media_resolution: nil,
+      output_audio_transcription: false,
+      &block
+    )
+      config = Configuration.new(
+        model: model,
+        response_modality: response_modality,
+        voice_name: voice_name,
+        system_instruction: system_instruction,
+        tools: tools,
+        context_window_compression: context_window_compression,
+        session_resumption: session_resumption,
+        automatic_activity_detection: automatic_activity_detection,
+        media_resolution: media_resolution,
+        output_audio_transcription: output_audio_transcription
+      )
+
+      session = Session.new(
+        api_key: @client.api_key,
+        configuration: config
+      )
+
+      if block_given?
+        begin
+          yield session
+        ensure
+          session.close
+        end
+      else
+        session
+      end
+    end
+  end
+end

data/lib/gemini/response.rb

@@ -41,9 +41,83 @@ module Gemini
     # Get image parts (if any)
     def image_parts
       return [] unless valid?
-      
+
       parts.select { |part| part.key?("inline_data") && part["inline_data"]["mime_type"].start_with?("image/") }
     end
+
+    # Get the first audio inlineData part (TTS responses use camelCase "inlineData")
+    def audio_part
+      return nil unless valid?
+
+      parts.find do |part|
+        data_key = part["inlineData"] || part["inline_data"]
+        next false unless data_key
+        mt = data_key["mimeType"] || data_key["mime_type"]
+        mt.is_a?(String) && mt.start_with?("audio/")
+      end
+    end
+
+    # Base64-encoded audio data from a TTS response
+    def audio_data
+      part = audio_part
+      return nil unless part
+      data_key = part["inlineData"] || part["inline_data"]
+      data_key["data"]
+    end
+
+    # MIME type of the audio payload (e.g. "audio/L16;codec=pcm;rate=24000")
+    def audio_mime_type
+      part = audio_part
+      return nil unless part
+      data_key = part["inlineData"] || part["inline_data"]
+      data_key["mimeType"] || data_key["mime_type"]
+    end
+
+    # True if the response contains audio inlineData
+    def audio_response?
+      !audio_part.nil?
+    end
+
+    # Save audio to a file. PCM (L16) payloads are wrapped in a WAV header so
+    # the result is directly playable; other audio MIME types are written as-is.
+    # Returns the written file path or nil if no audio is present.
+    def save_audio(filepath)
+      data_b64 = audio_data
+      return nil unless data_b64
+
+      require 'base64'
+      raw = Base64.strict_decode64(data_b64)
+      mime = audio_mime_type.to_s
+
+      if mime.include?("L16") || mime.include?("pcm")
+        rate = mime[/rate=(\d+)/, 1]&.to_i || 24000
+        channels = 1
+        bits_per_sample = 16
+        byte_rate = rate * channels * bits_per_sample / 8
+        block_align = channels * bits_per_sample / 8
+        data_size = raw.bytesize
+
+        header = +""
+        header << "RIFF"
+        header << [36 + data_size].pack("V")
+        header << "WAVE"
+        header << "fmt "
+        header << [16].pack("V")
+        header << [1].pack("v")
+        header << [channels].pack("v")
+        header << [rate].pack("V")
+        header << [byte_rate].pack("V")
+        header << [block_align].pack("v")
+        header << [bits_per_sample].pack("v")
+        header << "data"
+        header << [data_size].pack("V")
+
+        File.binwrite(filepath, header + raw)
+      else
+        File.binwrite(filepath, raw)
+      end
+      filepath
+    end
     
     # Get all content with string representation
     def full_content
@@ -70,9 +144,50 @@ module Gemini
     
     # Check if response is valid
     def valid?
-      !@raw_data.nil? && 
-      ((@raw_data.key?("candidates") && !@raw_data["candidates"].empty?) || 
-       (@raw_data.key?("predictions") && !@raw_data["predictions"].empty?))
+      !@raw_data.nil? &&
+      ((@raw_data.key?("candidates") && !@raw_data["candidates"].empty?) ||
+       (@raw_data.key?("predictions") && !@raw_data["predictions"].empty?) ||
+       embedding_response? ||
+       count_tokens_response?)
+    end
+
+    # Check if the raw response contains embedding data
+    def embedding_response?
+      return false if @raw_data.nil?
+      (@raw_data.key?("embedding") && !@raw_data["embedding"].nil?) ||
+        (@raw_data.key?("embeddings") && @raw_data["embeddings"].is_a?(Array) && !@raw_data["embeddings"].empty?)
+    end
+
+    # Get the embedding values as an Array of Floats.
+    # For single embedContent responses returns the values array.
+    # For batchEmbedContents responses returns the first embedding's values.
+    def embedding
+      return nil unless @raw_data
+      if @raw_data["embedding"].is_a?(Hash)
+        @raw_data["embedding"]["values"]
+      elsif @raw_data["embeddings"].is_a?(Array) && @raw_data["embeddings"].first.is_a?(Hash)
+        @raw_data["embeddings"].first["values"]
+      end
+    end
+
+    # Get all embedding value arrays for batch responses.
+    # Returns an Array of Arrays of Floats.
+    # For single embedContent responses, returns a single-element array.
+    def embeddings
+      return [] unless @raw_data
+      if @raw_data["embeddings"].is_a?(Array)
+        @raw_data["embeddings"].map { |e| e["values"] }.compact
+      elsif @raw_data["embedding"].is_a?(Hash) && @raw_data["embedding"]["values"]
+        [@raw_data["embedding"]["values"]]
+      else
+        []
+      end
+    end
+
+    # Get the dimensionality (length) of the first embedding vector
+    def embedding_dimension
+      values = embedding
+      values.is_a?(Array) ? values.length : 0
     end
     
     # Get error message if any
@@ -191,6 +306,28 @@ module Gemini
     def total_tokens
       usage&.dig("totalTokens") || 0
     end
+
+    # Check whether this response is a countTokens API result
+    def count_tokens_response?
+      !@raw_data.nil? && @raw_data.key?("totalTokens") &&
+        !@raw_data.key?("candidates") && !@raw_data.key?("predictions") &&
+        !embedding_response?
+    end
+
+    # Total tokens reported by the countTokens API (top-level totalTokens)
+    def count_tokens
+      @raw_data&.dig("totalTokens")
+    end
+
+    # Cached content token count reported by countTokens
+    def cached_content_token_count
+      @raw_data&.dig("cachedContentTokenCount") || 0
+    end
+
+    # Per-modality token breakdown reported by countTokens
+    def prompt_tokens_details
+      @raw_data&.dig("promptTokensDetails") || []
+    end
     
     # Process chunks for streaming responses
     def stream_chunks

data/lib/gemini/tokens.rb

@@ -0,0 +1,77 @@
+module Gemini
+  class Tokens
+    DEFAULT_MODEL = "gemini-2.5-flash".freeze
+
+    def initialize(client:)
+      @client = client
+    end
+
+    # Count tokens for the given input.
+    #
+    # input: String, Array of parts/contents, or Hash. Optional when `contents:` is given.
+    # contents: full Array of Content objects (overrides input).
+    # system_instruction: String or Content hash.
+    # tools: Array of tool definitions (passed via generateContentRequest form).
+    # generation_config: Hash forwarded as generationConfig.
+    # cached_content: cachedContents/* resource name.
+    def count(input = nil, model: DEFAULT_MODEL, contents: nil, system_instruction: nil,
+              tools: nil, generation_config: nil, cached_content: nil, **parameters)
+      normalized_model = normalize_model(model)
+
+      payload = build_payload(
+        model: normalized_model,
+        input: input,
+        contents: contents,
+        system_instruction: system_instruction,
+        tools: tools,
+        generation_config: generation_config,
+        cached_content: cached_content
+      ).merge(parameters)
+
+      response = @client.json_post(
+        path: "models/#{normalized_model}:countTokens",
+        parameters: payload
+      )
+      Gemini::Response.new(response)
+    end
+
+    private
+
+    def build_payload(model:, input:, contents:, system_instruction:, tools:, generation_config:, cached_content:)
+      resolved_contents = contents || [format_content(input)]
+
+      # Use generateContentRequest form when extra request fields are present
+      if system_instruction || tools || generation_config || cached_content
+        # model is required inside the nested GenerateContentRequest
+        gc_request = { model: "models/#{model}", contents: resolved_contents }
+        gc_request[:systemInstruction] = format_content(system_instruction) if system_instruction
+        gc_request[:tools] = tools if tools
+        gc_request[:generationConfig] = generation_config if generation_config
+        gc_request[:cachedContent] = cached_content if cached_content
+        { generateContentRequest: gc_request }
+      else
+        { contents: resolved_contents }
+      end
+    end
+
+    def format_content(input)
+      case input
+      when nil
+        raise ArgumentError, "input or contents parameter is required"
+      when String
+        { parts: [{ text: input }] }
+      when Array
+        { parts: input.map { |part| part.is_a?(String) ? { text: part } : part } }
+      when Hash
+        input.key?(:parts) || input.key?("parts") ? input : { parts: [input] }
+      else
+        { parts: [{ text: input.to_s }] }
+      end
+    end
+
+    def normalize_model(model)
+      model_str = model.to_s
+      model_str.start_with?("models/") ? model_str.delete_prefix("models/") : model_str
+    end
+  end
+end

data/lib/gemini/tts.rb

@@ -0,0 +1,83 @@
+module Gemini
+  class TTS
+    DEFAULT_MODEL = "gemini-2.5-flash-preview-tts".freeze
+
+    # 30 prebuilt voice names available for the prebuiltVoiceConfig
+    VOICES = %w[
+      Zephyr Puck Charon Kore Fenrir Leda Orus Aoede Callirrhoe Autonoe
+      Enceladus Iapetus Umbriel Algieba Despina Erinome Algenib Rasalgethi
+      Laomedeia Achernar Alnilam Schedar Gacrux Pulcherrima Achird
+      Zubenelgenubi Vindemiatrix Sadachbia Sadaltager Sulafat
+    ].freeze
+
+    def initialize(client:)
+      @client = client
+    end
+
+    # Generate speech audio from text.
+    #
+    # text: prompt String (use style cues / bracket tags like [excited] for control,
+    #       or "Speaker 1: ... Speaker 2: ..." for multi-speaker).
+    # voice: a single voice name (prebuiltVoiceConfig). Mutually exclusive with multi_speaker.
+    # multi_speaker: Array of { speaker:, voice: } Hashes for multi-speaker output.
+    # model: TTS preview model name. Defaults to gemini-2.5-flash-preview-tts.
+    # speech_config: raw speechConfig Hash override (skips voice/multi_speaker handling).
+    def generate(text, voice: nil, multi_speaker: nil, model: DEFAULT_MODEL,
+                 speech_config: nil, **parameters)
+      raise ArgumentError, "text is required" if text.nil? || text.to_s.empty?
+      if voice && multi_speaker
+        raise ArgumentError, "voice and multi_speaker are mutually exclusive"
+      end
+
+      resolved_speech_config = speech_config || build_speech_config(voice: voice, multi_speaker: multi_speaker)
+      raise ArgumentError, "voice, multi_speaker, or speech_config is required" unless resolved_speech_config
+
+      payload = {
+        contents: [{ parts: [{ text: text }] }],
+        generationConfig: {
+          responseModalities: ["AUDIO"],
+          speechConfig: resolved_speech_config
+        }
+      }
+
+      payload.merge!(parameters) if parameters && !parameters.empty?
+
+      response = @client.json_post(
+        path: "models/#{normalize_model(model)}:generateContent",
+        parameters: payload
+      )
+      Gemini::Response.new(response)
+    end
+
+    private
+
+    def build_speech_config(voice:, multi_speaker:)
+      if multi_speaker
+        speaker_voice_configs = multi_speaker.map do |entry|
+          speaker = entry[:speaker] || entry["speaker"]
+          v = entry[:voice] || entry["voice"]
+          raise ArgumentError, "multi_speaker entries require :speaker and :voice" unless speaker && v
+          validate_voice!(v)
+          {
+            speaker: speaker,
+            voiceConfig: { prebuiltVoiceConfig: { voiceName: v } }
+          }
+        end
+        { multiSpeakerVoiceConfig: { speakerVoiceConfigs: speaker_voice_configs } }
+      elsif voice
+        validate_voice!(voice)
+        { voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } } }
+      end
+    end
+
+    def validate_voice!(voice)
+      return if VOICES.include?(voice.to_s)
+      raise ArgumentError, "Unknown voice '#{voice}'. Available voices: #{VOICES.join(', ')}"
+    end
+
+    def normalize_model(model)
+      model_str = model.to_s
+      model_str.start_with?("models/") ? model_str.delete_prefix("models/") : model_str
+    end
+  end
+end

data/lib/gemini/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Gemini
-  VERSION = "1.0.0"
+  VERSION = "1.2.0"
 end

data/lib/gemini.rb

@@ -12,6 +12,8 @@ require_relative "gemini/threads"
 require_relative "gemini/messages"
 require_relative "gemini/runs"
 require_relative "gemini/embeddings"
+require_relative "gemini/tokens"
+require_relative "gemini/tts"
 require_relative "gemini/audio"
 require_relative "gemini/files"
 require_relative "gemini/images"
@@ -20,6 +22,7 @@ require_relative "gemini/function_calling_helper"
 require_relative "gemini/documents"
 require_relative "gemini/cached_content"
 require_relative "gemini/video"
+require_relative "gemini/live"
 
 module Gemini
   class Error < StandardError; end